{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "v4XSlX_RMQuq"
      },
      "outputs": [],
      "source": [
        "# Copyright 2021 The Google Research Authors.\n",
        "#\n",
        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
        "# you may not use this file except in compliance with the License.\n",
        "# You may obtain a copy of the License at\n",
        "#\n",
        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
        "#\n",
        "# Unless required by applicable law or agreed to in writing, software\n",
        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
        "# See the License for the specific language governing permissions and\n",
        "# limitations under the License."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CHVbLYfAjE-o"
      },
      "source": [
        "# NOTE\n",
        "#### Make sure that this notebook is using `smug` kernel"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "T7x4V6xhFlGD",
        "outputId": "89758dd3-d562-4b8d-a02d-15768c3643a9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "annotations.json    100%[===================\u003e]   2.21M   798KB/s    in 2.8s    \n",
            "review+wiki.filtere 100%[===================\u003e]  95.50M   435KB/s    in 2m 51s  \n",
            "reviews.260k.heldou 100%[===================\u003e]   7.70M   457KB/s    in 25s     \n",
            "reviews.260k.train. 100%[===================\u003e]  56.66M  1.12MB/s    in 89s     \n",
            "reviews.aspect0.hel 100%[===================\u003e]   2.56M   617KB/s    in 4.2s    \n",
            "reviews.aspect0.tra 100%[===================\u003e]  20.20M   451KB/s    in 52s     \n",
            "reviews.aspect1.hel 100%[===================\u003e]   2.54M   436KB/s    in 6.9s    \n",
            "reviews.aspect1.tra 100%[===================\u003e]  17.87M   404KB/s    in 52s     \n",
            "reviews.aspect2.hel 100%[===================\u003e]   2.59M   530KB/s    in 6.4s    \n",
            "reviews.aspect2.tra 100%[===================\u003e]  18.19M   738KB/s    in 20s     \n"
          ]
        }
      ],
      "source": [
        "import os\n",
        "if not os.path.exists('data'):\n",
        "  ! mkdir data\n",
        "  ! wget wget -q -P data/ --show-progress http://people.csail.mit.edu/taolei/beer/annotations.json\n",
        "  ! wget wget -q -P data/ --show-progress http://people.csail.mit.edu/taolei/beer/review+wiki.filtered.200.txt.gz\n",
        "  ! wget wget -q -P data/ --show-progress http://people.csail.mit.edu/taolei/beer/reviews.260k.heldout.txt.gz\n",
        "  ! wget wget -q -P data/ --show-progress http://people.csail.mit.edu/taolei/beer/reviews.260k.train.txt.gz\n",
        "  ! wget wget -q -P data/ --show-progress http://people.csail.mit.edu/taolei/beer/reviews.aspect0.heldout.txt.gz\n",
        "  ! wget wget -q -P data/ --show-progress http://people.csail.mit.edu/taolei/beer/reviews.aspect0.train.txt.gz\n",
        "  ! wget wget -q -P data/ --show-progress http://people.csail.mit.edu/taolei/beer/reviews.aspect1.heldout.txt.gz\n",
        "  ! wget wget -q -P data/ --show-progress http://people.csail.mit.edu/taolei/beer/reviews.aspect1.train.txt.gz\n",
        "  ! wget wget -q -P data/ --show-progress http://people.csail.mit.edu/taolei/beer/reviews.aspect2.heldout.txt.gz\n",
        "  ! wget wget -q -P data/ --show-progress http://people.csail.mit.edu/taolei/beer/reviews.aspect2.train.txt.gz"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rERcG7e8FlGJ"
      },
      "outputs": [],
      "source": [
        "import gzip\n",
        "import json\n",
        "\n",
        "from IPython.display import clear_output, display, Image, HTML\n",
        "from matplotlib import pyplot as plt\n",
        "import numpy as np\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "from tensorflow.compat.v1 import gfile\n",
        "from tensorflow.compat.v1 import keras\n",
        "import tensorflow.compat.v1 as tf\n",
        "from tensorflow.compat.v1.keras.preprocessing import sequence, text\n",
        "\n",
        "tf.disable_eager_execution()\n",
        "\n",
        "os.chdir('../')\n",
        "from smug_saliency import masking\n",
        "from smug_saliency import utils\n",
        "os.chdir('smug_saliency/')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FvUsBfMFFlGJ"
      },
      "source": [
        "### Train 1D CNN"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aVJL2_shFlGK"
      },
      "outputs": [],
      "source": [
        "def load_reviews(path, verbose=True):\n",
        "  data_x, data_y = [ ], [ ]\n",
        "  fopen = gfile.Open(path, 'rb')\n",
        "  if path.endswith(\".gz\"):\n",
        "    fopen = gzip.GzipFile(fileobj=fopen)\n",
        "\n",
        "  for line in fopen:\n",
        "    line = line.decode('ascii')\n",
        "    y, sep, x = line.partition(\"\\t\")\n",
        "    # x = x.split()\n",
        "    y = y.split()\n",
        "    if len(x) == 0: continue\n",
        "    y = np.asarray([ float(v) for v in y ])\n",
        "    data_x.append(x)\n",
        "    data_y.append(y)\n",
        "  fopen.close()\n",
        "\n",
        "  if verbose:\n",
        "    print(\"{} examples loaded from {}\".format(len(data_x), path))\n",
        "    print(\"max text length: {}\".format(max(len(x) for x in data_x)))\n",
        "\n",
        "  return data_x, data_y\n",
        "\n",
        "def create_splits(X, y, test_size=3000, random_state=42):\n",
        "  return train_test_split(X,\n",
        "                          y,\n",
        "                          test_size=3000,\n",
        "                          random_state=42)\n",
        "\n",
        "def create_tokenizer(X, top_words=10000):\n",
        "    tokenizer = text.Tokenizer(num_words=top_words)\n",
        "    tokenizer.fit_on_texts(X)\n",
        "    return tokenizer\n",
        "\n",
        "def pad_sequences(X, max_words=500):\n",
        "    return sequence.pad_sequences(X, maxlen=max_words)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hlbbL7blFlGK",
        "outputId": "eccae023-3c90-4a5b-9d03-afb15fa930e2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "70000 examples loaded from data/reviews.aspect1.train.txt.gz\n",
            "max text length: 5085\n",
            "10000 examples loaded from data/reviews.aspect1.heldout.txt.gz\n",
            "max text length: 4619\n"
          ]
        }
      ],
      "source": [
        "ASPECT = 1  # 0, 1, 2, or 3\n",
        "BASE_PATH = 'data'\n",
        "\n",
        "path = os.path.join(BASE_PATH, 'reviews.aspect' + str(ASPECT))\n",
        "train_path = path + '.train.txt.gz'\n",
        "heldout_path = path + '.heldout.txt.gz'\n",
        "\n",
        "X_train_texts, y_train = load_reviews(train_path)\n",
        "X_test_texts, y_test = load_reviews(heldout_path)\n",
        "\n",
        "# y value is just the sentiment for this aspect, throw away the other scores\n",
        "y_train = np.array([y[ASPECT] for y in y_train])\n",
        "y_test = np.array([y[ASPECT] for y in y_test])\n",
        "\n",
        "# Create a 3k validation set held-out from the test set\n",
        "X_test_texts, X_val_texts, y_test, y_val = create_splits(X_test_texts, y_test)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JaBO6CVVFlGL",
        "outputId": "22f15339-85cd-493a-ad33-93e201a04553",
        "scrolled": true
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD4CAYAAAAO9oqkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAASWklEQVR4nO3de5CddX3H8fdHUrxVDMKWwSQ1aY3awNgRd4COM2rFgYDWoKITWmuwqRktXlp1FGpncFCmUjtQGdFOCqmBUQJSO6QFpSnCMDoGWAS5iqwgkshlNQHbMl5iv/3j/FIPcTfZ3bN79pC8XzNn9nm+z+8553tONudznst5NlWFJGnf9rS5bkCSNPcMA0mSYSBJMgwkSRgGkiRg3lw3MF0HH3xwLV68eK7bkKSnlJtvvvlHVTW0a/0pGwaLFy9mZGRkrtuQpKeUJA+MV3c3kSTJMJAkGQaSJCYRBknWJXk0yR1dtU8l+U6S25L8a5L5XctOTzKa5J4kx3XVl7faaJLTuupLktzQ6pcm2X8mn6Akac8ms2XweWD5LrVNwOFV9VLgu8DpAEmWASuBw9o6n02yX5L9gPOB44FlwMltLMDZwLlV9UJgO7C6p2ckSZqyPYZBVV0PbNul9h9VtaPNbgYWtukVwIaq+llV3Q+MAke222hV3VdVPwc2ACuSBHgNcHlbfz1wYo/PSZI0RTNxzODPgK+06QXAg13LtrTaRPWDgMe6gmVnfVxJ1iQZSTIyNjY2A61LkqDHMEjyUWAH8IWZaWf3qmptVQ1X1fDQ0K99Z0KSNE3T/tJZklOA1wPH1K/+KMJWYFHXsIWtxgT1HwPzk8xrWwfd4yVJfTKtMEiyHPgw8KqqeqJr0Ubgi0nOAZ4PLAVuBAIsTbKEzpv9SuCPq6qSXAucROc4wirgiuk+GUnql8WnXTknj/v9T75uVu53MqeWXgJ8E3hxki1JVgOfAZ4DbEpya5J/BKiqO4HLgLuArwKnVtUv26f+9wBXA3cDl7WxAB8BPpBklM4xhAtn9BlKkvZoj1sGVXXyOOUJ37Cr6izgrHHqVwFXjVO/j87ZRpKkOeI3kCVJhoEkyTCQJGEYSJIwDCRJGAaSJJ7Cf/ZSkubqi197I7cMJEmGgSTJMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKE30DWXmxv+7OE0mxyy0CSZBhIkgwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSUwiDJKsS/Jokju6as9LsinJve3nga2eJOclGU1yW5IjutZZ1cbfm2RVV/3lSW5v65yXJDP9JCVJuzeZLYPPA8t3qZ0GXFNVS4Fr2jzA8cDSdlsDfA464QGcARwFHAmcsTNA2ph3dq2362NJkmbZHsOgqq4Htu1SXgGsb9PrgRO76hdVx2ZgfpJDgeOATVW1raq2A5uA5W3ZAVW1uaoKuKjrviRJfTLdYwaHVNVDbfph4JA2vQB4sGvcllbbXX3LOPVxJVmTZCTJyNjY2DRblyTtqucDyO0Tfc1AL5N5rLVVNVxVw0NDQ/14SEnaJ0z3EtaPJDm0qh5qu3oebfWtwKKucQtbbSvw6l3q17X6wnHGa4Z5OWdJuzPdLYONwM4zglYBV3TV397OKjoaeLztTroaODbJge3A8bHA1W3ZT5Ic3c4ienvXfUmS+mSPWwZJLqHzqf7gJFvonBX0SeCyJKuBB4C3tuFXAScAo8ATwDsAqmpbko8DN7VxZ1bVzoPSf0HnjKVnAl9pN0lSH+0xDKrq5AkWHTPO2AJOneB+1gHrxqmPAIfvqQ9J0uzxG8iSJMNAkmQYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkMYk/eylJe7L4tCvnugX1yC0DSZJhIElyN5FmmbsPpKcGtwwkSYaBJMkwkCRhGEiS6DEMkvxVkjuT3JHkkiTPSLIkyQ1JRpNcmmT/NvbpbX60LV/cdT+nt/o9SY7r7SlJkqZq2mGQZAHwPmC4qg4H9gNWAmcD51bVC4HtwOq2ympge6uf28aRZFlb7zBgOfDZJPtNty9J0tT1uptoHvDMJPOAZwEPAa8BLm/L1wMntukVbZ62/JgkafUNVfWzqrofGAWO7LEvSdIUTDsMqmor8PfAD+iEwOPAzcBjVbWjDdsCLGjTC4AH27o72viDuuvjrPMkSdYkGUkyMjY2Nt3WJUm76GU30YF0PtUvAZ4PPJvObp5ZU1Vrq2q4qoaHhoZm86EkaZ/Sy26i1wL3V9VYVf0C+DLwCmB+220EsBDY2qa3AosA2vLnAj/uro+zjiSpD3oJgx8ARyd5Vtv3fwxwF3AtcFIbswq4ok1vbPO05V+rqmr1le1soyXAUuDGHvqSJE3RtK9NVFU3JLkc+BawA7gFWAtcCWxI8olWu7CtciFwcZJRYBudM4ioqjuTXEYnSHYAp1bVL6fblyRp6nq6UF1VnQGcsUv5PsY5G6iqfgq8ZYL7OQs4q5deJEnT5zeQJUmGgSTJMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEmixzBIMj/J5Um+k+TuJH+Q5HlJNiW5t/08sI1NkvOSjCa5LckRXfezqo2/N8mqXp+UJGlqet0y+DTw1ap6CfD7wN3AacA1VbUUuKbNAxwPLG23NcDnAJI8DzgDOAo4EjhjZ4BIkvpj2mGQ5LnAK4ELAarq51X1GLACWN+GrQdObNMrgIuqYzMwP8mhwHHApqraVlXbgU3A8un2JUmaul62DJYAY8A/J7klyQVJng0cUlUPtTEPA4e06QXAg13rb2m1ieq/JsmaJCNJRsbGxnpoXZLUrZcwmAccAXyuql4G/A+/2iUEQFUVUD08xpNU1dqqGq6q4aGhoZm6W0na5/USBluALVV1Q5u/nE44PNJ2/9B+PtqWbwUWda2/sNUmqkuS+mTaYVBVDwMPJnlxKx0D3AVsBHaeEbQKuKJNbwTe3s4qOhp4vO1Ouho4NsmB7cDxsa0mSeqTeT2u/17gC0n2B+4D3kEnYC5Lshp4AHhrG3sVcAIwCjzRxlJV25J8HLipjTuzqrb12JckaQp6CoOquhUYHmfRMeOMLeDUCe5nHbCul14kSdPnN5AlSYaBJMkwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgSWIGwiDJfkluSfLvbX5JkhuSjCa5NMn+rf70Nj/ali/uuo/TW/2eJMf12pMkaWpmYsvg/cDdXfNnA+dW1QuB7cDqVl8NbG/1c9s4kiwDVgKHAcuBzybZbwb6kiRNUk9hkGQh8DrggjYf4DXA5W3IeuDENr2izdOWH9PGrwA2VNXPqup+YBQ4spe+JElT0+uWwT8AHwb+t80fBDxWVTva/BZgQZteADwI0JY/3sb/f32cdZ4kyZokI0lGxsbGemxdkrTTtMMgyeuBR6vq5hnsZ7eqam1VDVfV8NDQUL8eVpL2evN6WPcVwBuSnAA8AzgA+DQwP8m89ul/IbC1jd8KLAK2JJkHPBf4cVd9p+51JEl9MO0wqKrTgdMBkrwa+FBV/UmSLwEnARuAVcAVbZWNbf6bbfnXqqqSbAS+mOQc4PnAUuDG6fYl7csWn3blXLegp6hetgwm8hFgQ5JPALcAF7b6hcDFSUaBbXTOIKKq7kxyGXAXsAM4tap+OQt9SZImMCNhUFXXAde16fsY52ygqvop8JYJ1j8LOGsmepEkTZ3fQJYkGQaSJMNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJLE7FyoThPwipKSBpVbBpIkw0CSZBhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJPbRq5Z69VBJerJpbxkkWZTk2iR3Jbkzyftb/XlJNiW5t/08sNWT5Lwko0luS3JE132tauPvTbKq96clSZqKXnYT7QA+WFXLgKOBU5MsA04DrqmqpcA1bR7geGBpu60BPged8ADOAI4CjgTO2BkgkqT+mHYYVNVDVfWtNv1fwN3AAmAFsL4NWw+c2KZXABdVx2ZgfpJDgeOATVW1raq2A5uA5dPtS5I0dTNyADnJYuBlwA3AIVX1UFv0MHBIm14APNi12pZWm6g+3uOsSTKSZGRsbGwmWpckMQNhkOQ3gX8B/rKqftK9rKoKqF4fo+v+1lbVcFUNDw0NzdTdStI+r6cwSPIbdILgC1X15VZ+pO3+of18tNW3Aou6Vl/YahPVJUl90svZRAEuBO6uqnO6Fm0Edp4RtAq4oqv+9nZW0dHA42130tXAsUkObAeOj201SVKf9PI9g1cAfwrcnuTWVvtr4JPAZUlWAw8Ab23LrgJOAEaBJ4B3AFTVtiQfB25q486sqm099CVJmqJph0FVfR3IBIuPGWd8AadOcF/rgHXT7UWS1BsvRyFJMgwkSYaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEgMUBkmWJ7knyWiS0+a6H0nalwxEGCTZDzgfOB5YBpycZNncdiVJ+46BCAPgSGC0qu6rqp8DG4AVc9yTJO0z5s11A80C4MGu+S3AUbsOSrIGWNNm/zvJPdN8vIOBH01z3dlkX1MzkH3l7MHsiwF9vbCvKZmB368XjFcclDCYlKpaC6zt9X6SjFTV8Ay0NKPsa2rsa2rsa2r2tb4GZTfRVmBR1/zCVpMk9cGghMFNwNIkS5LsD6wENs5xT5K0zxiI3URVtSPJe4Crgf2AdVV15yw+ZM+7mmaJfU2NfU2NfU3NPtVXqmo27leS9BQyKLuJJElzyDCQJO3dYbCnS1wk+UCSu5LcluSaJOOefzsHfb0rye1Jbk3y9X59G3uylwRJ8uYklaQvp91N4vU6JclYe71uTfLng9BXG/PW9jt2Z5IvDkJfSc7teq2+m+SxAenrt5Ncm+SW9n/yhAHp6wXt/eG2JNclWdiHntYleTTJHRMsT5LzWs+3JTmi5wetqr3yRudA9PeA3wH2B74NLNtlzB8Cz2rT7wYuHZC+DuiafgPw1UHoq417DnA9sBkYHoS+gFOAzwzg79dS4BbgwDb/W4PQ1y7j30vnhI0574vOgdF3t+llwPcHpK8vAava9GuAi/vQ1yuBI4A7Jlh+AvAVIMDRwA29PubevGWwx0tcVNW1VfVEm91M5/sNg9DXT7pmnw304yj/ZC8J8nHgbOCnfehpKn3122T6eidwflVtB6iqRwekr24nA5cMSF8FHNCmnwv8cED6WgZ8rU1fO87yGVdV1wPbdjNkBXBRdWwG5ic5tJfH3JvDYLxLXCzYzfjVdJJ2tk2qrySnJvke8HfA+wahr7YpuqiqruxDP5Puq3lz21y+PMmicZbPRV8vAl6U5BtJNidZPiB9AZ3dH8ASfvVGN9d9fQx4W5ItwFV0tloGoa9vA29q028EnpPkoD70tjtTfX/bo705DCYtyduAYeBTc93LTlV1flX9LvAR4G/mup8kTwPOAT44172M49+AxVX1UmATsH6O+9lpHp1dRa+m8wn8n5LMn9OOnmwlcHlV/XKuG2lOBj5fVQvp7Aa5uP3ezbUPAa9KcgvwKjpXRxiU12zGDMILPVsmdYmLJK8FPgq8oap+Nih9ddkAnDirHXXsqa/nAIcD1yX5Pp39lBv7cBB5j69XVf2469/uAuDls9zTpPqi82ltY1X9oqruB75LJxzmuq+dVtKfXUQwub5WA5cBVNU3gWfQuVjcnPZVVT+sqjdV1cvovFdQVX056L4bM38Jn9k+EDJXNzqfyu6jsxm888DQYbuMeRmdg0dLB6yvpV3TfwSMDEJfu4y/jv4cQJ7M63Vo1/Qbgc0D0tdyYH2bPpjOZv1Bc91XG/cS4Pu0L54OyOv1FeCUNv17dI4ZzGp/k+zrYOBpbfos4Mw+vWaLmfgA8ut48gHkG3t+vH48qbm60dnU/G57w/9oq51JZysA4D+BR4Bb223jgPT1aeDO1tO1u3tT7mdfu4ztSxhM8vX62/Z6fbu9Xi8ZkL5CZ9faXcDtwMpB6KvNfwz4ZD/6mcLrtQz4Rvt3vBU4dkD6Ogm4t425AHh6H3q6BHgI+AWdLczVwLuAd3X9bp3fer59Jv4vejkKSdJefcxAkjRJhoEkyTCQJBkGkiQMA0kShoEkCcNAkgT8H3UpMQU7Vd0hAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "\u003cFigure size 432x288 with 1 Axes\u003e"
            ]
          },
          "metadata": {
            "needs_background": "light",
            "tags": []
          },
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Unique y values:  [0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 1. ]\n",
            "Mean y value:  0.6802100000000001\n",
            "Median y value:  0.7\n",
            "Std y value:  0.23792000195143623\n"
          ]
        }
      ],
      "source": [
        "plt.hist(y_train)\n",
        "plt.show()\n",
        "\n",
        "print('Unique y values: ', np.unique(y_train))\n",
        "print('Mean y value: ', np.mean(y_train))\n",
        "print('Median y value: ', np.median(y_train))\n",
        "print('Std y value: ', np.std(y_train))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7QE9pTUKFlGL",
        "outputId": "c3bccfab-7a1f-4c2e-e25e-6ca31314811b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Review length:\n",
            "Mean 154.47 words (stddev: 81.134751)\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAARjElEQVR4nO3df2xdZ33H8fc3N449UpEmrVXR/FgrETFn1iYqq3QlmgidoGWI5A+GGqGRMkvRpOKxdRKU+Y+yTZVAm9ZBxSpFJCNIyFB1jEZTtaxqjJDF2pECKmkNq1VU4tCSlIQy1UpqO9/94ePUCWnie69zb26e90uy7jnPec49X0fJ55485znnRmYiSSrDsnYXIElqHUNfkgpi6EtSQQx9SSqIoS9JBTH0JakgFw39iNgTEUcj4tCCtn+IiB9HxDMR8e8RcfWCbZ+JiImI+ElEvH9B++1V20RE3Lv0v4ok6WIWc6b/FeD2c9oeB/oz8/eA/wU+AxARm4A7gd+t9vmXiKhFRA34EnAHsAnYXvWVJLXQRUM/M78DHD+n7b8yc6ZafRJYVy1vBb6emacy86fABHBz9TORmS9k5uvA16u+kqQWWr4E7/FnwDeq5bXMfQjMm6zaAA6f0/6ui73xtddemzfccMMSlChJ5Xj66adfycze821rKvQjYhiYAb7WzPuc8547gZ0AGzZs4ODBg0v11pJUhIh48c22NTx7JyLuAj4IfDTfeIDPEWD9gm7rqrY3a/8NmbkrMwcyc6C397wfVJKkBjUU+hFxO/Ap4EOZObVg0z7gzojojogbgY3A/wDfAzZGxI0RsYK5i737mitdklSviw7vRMQI8B7g2oiYBO5jbrZON/B4RAA8mZl/npnPRsTDwHPMDfvcnZmz1ft8AtgP1IA9mfnsJfh9JEkXEJfzo5UHBgbSMX1Jqk9EPJ2ZA+fb5h25klQQQ1+q08jICP39/dRqNfr7+xkZGWl3SdKiLcU8fakYIyMjDA8Ps3v3bjZv3szY2BiDg4MAbN++vc3VSRfnmL5Uh/7+fh588EG2bNlypm10dJShoSEOHTp0gT2l1rnQmL6hL9WhVqtx8uRJurq6zrRNT0/T09PD7OxsGyuT3uCFXGmJ9PX1MTY2dlbb2NgYfX19bapIqo+hL9VheHiYwcFBRkdHmZ6eZnR0lMHBQYaHh9tdmrQoXsiV6jB/sXZoaIjx8XH6+vq4//77vYirjuGYviRdYRzTlyQBhr4kFcXQl6SCGPqSVBBDX5IKYuhLUkEMfUkqiKEvSQUx9CWpIIa+JBXE0Jekghj6klQQQ1+SCmLoS1JBDH2pTiMjI/T391Or1ejv72dkZKTdJUmL5peoSHUYGRlheHiY3bt3s3nzZsbGxhgcHATwi1TUEfwSFakO/f39PPjgg2zZsuVM2+joKENDQxw6dKiNlUlvaOpLVCJiT0QcjYhDC9rWRMTjEfF89bq6ao+I+GJETETEMxFx04J9dlT9n4+IHUvxi0mtNj4+zubNm89q27x5M+Pj422qSKrPYsb0vwLcfk7bvcATmbkReKJaB7gD2Fj97AQegrkPCeA+4F3AzcB98x8UUifp6+tjbGzsrLaxsTH6+vraVJFUn4uGfmZ+Bzh+TvNWYG+1vBfYtqD9qznnSeDqiHgb8H7g8cw8npkngMf5zQ8S6bI3PDzM4OAgo6OjTE9PMzo6yuDgIMPDw+0uTVqURi/kXpeZL1XLLwPXVctrgcML+k1WbW/W/hsiYidz/0tgw4YNDZYnXRrzF2uHhoYYHx+nr6+P+++/34u46hhNz97JzIyIJbsanJm7gF0wdyF3qd5XWirbt2835NWxGp2n/4tq2Ibq9WjVfgRYv6DfuqrtzdolSS3UaOjvA+Zn4OwAHl3Q/rFqFs8twKvVMNB+4H0Rsbq6gPu+qk2S1EIXHd6JiBHgPcC1ETHJ3CyczwEPR8Qg8CLwkar7Y8AHgAlgCvg4QGYej4i/B75X9fu7zDz34rAk6RLz5ixJusI0dXOWJOnKYehLUkEMfUkqiKEv1clHK6uT+WhlqQ4+Wlmdztk7Uh18tLI6wYVm7xj6Uh1qtRonT56kq6vrTNv09DQ9PT3Mzs62sTLpDU7ZlJaIj1ZWpzP0pTr4aGV1Oi/kSnXw0crqdI7pS9IVxjF9SRJg6Et18+YsdTLH9KU6eHOWOp1j+lId+vv72bZtG9/61rfOXMidX/fmLF0uLjSm75m+VIfnnnuOo0ePsnLlSjKT1157jV27dvHKK6+0uzRpURzTl+pQq9WYmZlhz549nDp1ij179jAzM0OtVmt3adKiGPpSHWZmZuju7j6rrbu7m5mZmTZVJNXH0JfqdNdddzE0NERPTw9DQ0Pcdddd7S5JWjQv5Ep1WL9+PcePH2d6eprp6Wm6urro6upizZo1HD58uN3lSYA3Z0lLZtOmTUxNTZ15oubs7CxTU1Ns2rSpzZVJi+PsHakOBw4coLu7m9OnT3P69GlqtRpdXV0cOHCg3aVJi+KZvlSHmZkZVq1axf79+3n99dfZv38/q1at8kKuOoahL9Vp27ZtbNmyha6uLrZs2cK2bdvaXZK0aF7IleoQEUQEy5YtY3Z2llqtxunTp8lMLud/SyrLJbuQGxF/FRHPRsShiBiJiJ6IuDEinoqIiYj4RkSsqPp2V+sT1fYbmjm21A7zd+LOB/z88sqVK9tcmbQ4DYd+RKwF/gIYyMx+oAbcCXweeCAz3w6cAAarXQaBE1X7A1U/qaOcOnWK7u7uM3fg1mo1uru7OXXqVJsrkxan2TH95cBvRcRy4C3AS8B7gUeq7XuB+QHPrdU61fbbIiKaPL7UUjMzM6xYseKsthUrVnghVx2j4dDPzCPAPwI/Yy7sXwWeBn6VmfP/AiaBtdXyWuBwte9M1f+aRo8vtcvU1BTT09MATE9PMzU11eaKpMVrZnhnNXNn7zcC1wMrgdubLSgidkbEwYg4eOzYsWbfTlpys7Oz9PT0ANDT03PmRi2pEzQzvPNHwE8z81hmTgPfBN4NXF0N9wCsA45Uy0eA9QDV9lXAL89908zclZkDmTnQ29vbRHnSpTE/cwfmPgCWLXPmszpHM39bfwbcEhFvqcbmbwOeA0aBD1d9dgCPVsv7qnWq7QfSOW7qQF1dXRdcly5nzYzpP8XcBdnvAz+q3msX8GngnoiYYG7Mfne1y27gmqr9HuDeJuqW2ubUqVNcddVVLFu2jKuuusqZO+oo3pwl1WF+wtmyZcs4ffr0mVfAm7N02fApm9ISmw/6+VepUxj6Uh0ignXr1p054z93XbrcGfpSHTKTycnJs0J/cnLSoR11DENfasDC0Jc6iaEvNWDhPH2pkxj6klQQQ19qwK233srPf/5zbr311naXItXF78iVGvDd736X66+/vt1lSHXzTF9qwPLly896lTqFoS81wAu56lSGvtSAhV+XKHUSQ1+SCmLoS1JBDH1JKoihLzVg4dclSp3E0JcacPLkybNepU5h6EtSQQx9SSqIoS9JBTH0Jakghr4kFcTQl6SCGPqSVBBDX5IKYuhLUkEMfUkqSFOhHxFXR8QjEfHjiBiPiD+IiDUR8XhEPF+9rq76RkR8MSImIuKZiLhpaX4FSdJiNXum/wXgPzPzd4DfB8aBe4EnMnMj8ES1DnAHsLH62Qk81OSxJUl1ajj0I2IV8IfAboDMfD0zfwVsBfZW3fYC26rlrcBXc86TwNUR8baGK5ck1a2ZM/0bgWPAv0bEDyLiyxGxErguM1+q+rwMXFctrwUOL9h/smqTJLVIM6G/HLgJeCgz3wm8xhtDOQDk3BeI1vUlohGxMyIORsTBY8eONVGeJOlczYT+JDCZmU9V648w9yHwi/lhm+r1aLX9CLB+wf7rqrazZOauzBzIzIHe3t4mypMknavh0M/Ml4HDEfGOquk24DlgH7CjatsBPFot7wM+Vs3iuQV4dcEwkCSpBZY3uf8Q8LWIWAG8AHycuQ+ShyNiEHgR+EjV9zHgA8AEMFX1lSS1UFOhn5k/BAbOs+m28/RN4O5mjidJao535EpSQQx9SSqIoS9JBTH0Jakghr4kFcTQl6SCGPqSVBBDX5IKYuhLUkEMfUkqiKEvSQUx9CWpIIa+JBXE0Jekghj6klQQQ1+SCmLoS1JBDH1JKoihL0kFMfQlqSCGviQVxNCXpIIY+pJUEENfkgpi6EtSQQx9SSpI06EfEbWI+EFE/Ee1fmNEPBURExHxjYhYUbV3V+sT1fYbmj22JKk+S3Gm/0lgfMH654EHMvPtwAlgsGofBE5U7Q9U/SRJLdRU6EfEOuCPgS9X6wG8F3ik6rIX2FYtb63WqbbfVvWXJLVIs2f6/wx8CjhdrV8D/CozZ6r1SWBttbwWOAxQbX+16n+WiNgZEQcj4uCxY8eaLE+StFDDoR8RHwSOZubTS1gPmbkrMwcyc6C3t3cp31qSire8iX3fDXwoIj4A9ABvBb4AXB0Ry6uz+XXAkar/EWA9MBkRy4FVwC+bOL4kqU4Nn+ln5mcyc11m3gDcCRzIzI8Co8CHq247gEer5X3VOtX2A5mZjR5fklS/SzFP/9PAPRExwdyY/e6qfTdwTdV+D3DvJTi2JOkCmhneOSMzvw18u1p+Abj5PH1OAn+yFMeTJDXGO3IlqSCGviQVxNCXpIIY+pJUEENfkgpi6EtSQZZkyqbU6Zbi2X+LeQ/vR1S7GfoSiw/jCwW7ga5O4PCOJBXE0Jfq8GZn857lq1M4vCPVaT7gI8KwV8fxTF+SCmLoS1JBDH1JKoihL0kFMfQlqSCGviQVxNCXpIIY+pJUEENfkgpi6EtSQQx9SSqIoS9JBTH0Jakghr4kFaTh0I+I9RExGhHPRcSzEfHJqn1NRDweEc9Xr6ur9oiIL0bEREQ8ExE3LdUvIUlanGbO9GeAv87MTcAtwN0RsQm4F3giMzcCT1TrAHcAG6ufncBDTRxbktSAhkM/M1/KzO9Xy/8HjANrga3A3qrbXmBbtbwV+GrOeRK4OiLe1nDlkqS6LcmYfkTcALwTeAq4LjNfqja9DFxXLa8FDi/YbbJqkyS1SNOhHxFXAf8G/GVm/nrhtpz7Lrm6vk8uInZGxMGIOHjs2LFmy5MkLdBU6EdEF3OB/7XM/GbV/Iv5YZvq9WjVfgRYv2D3dVXbWTJzV2YOZOZAb29vM+VJks7RzOydAHYD45n5Tws27QN2VMs7gEcXtH+smsVzC/DqgmEgaUmtWbOGiLikP8AlP8aaNWva/CepK83yJvZ9N/CnwI8i4odV298AnwMejohB4EXgI9W2x4APABPAFPDxJo4tXdCJEyeYG13sbPMfLtJSaTj0M3MMeLO/kbedp38Cdzd6PElS87wjV5IKYuhLUkEMfUkqiKEvSQUx9CWpIIa+JBWkmXn60mUr73srfHZVu8toWt731naXoCuMoa8rUvztr6+Ym7Pys+2uQlcSh3ckqSCGviQVxNCXpIIY+pJUEENfkgri7B1dsa6ExxKvXr263SXoCmPo64rUiumaEXFFTAtVWRzekaSCGPqSVBBDX5IKYuhLUkEMfUkqiKEvSQUx9CWpIIa+JBXE0Jekghj6klQQQ1+SCtLy0I+I2yPiJxExERH3tvr4klSyloZ+RNSALwF3AJuA7RGxqZU1SFLJWn2mfzMwkZkvZObrwNeBrS2uQZKK1epHK68FDi9YnwTetbBDROwEdgJs2LChdZWpaI0+e7/e/XwUs9rtsruQm5m7MnMgMwd6e3vbXY4KkZkt+ZHardWhfwRYv2B9XdUmSWqBVof+94CNEXFjRKwA7gT2tbgGSSpWS8f0M3MmIj4B7AdqwJ7MfLaVNUhSyVr+HbmZ+RjwWKuPK0m6DC/kSpIuHUNfkgpi6EtSQQx9SSpIXM43jETEMeDFdtchvYlrgVfaXYR0Hr+dmee9u/WyDn3pchYRBzNzoN11SPVweEeSCmLoS1JBDH2pcbvaXYBUL8f0JakgnulLUkEMfalOEbEnIo5GxKF21yLVy9CX6vcV4PZ2FyE1wtCX6pSZ3wGOt7sOqRGGviQVxNCXpIIY+pJUEENfkgpi6Et1iogR4L+Bd0TEZEQMtrsmabG8I1eSCuKZviQVxNCXpIIY+pJUEENfkgpi6EtSQQx9SSqIoS9JBTH0Jakg/w+Qep1dsBL5HwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "\u003cFigure size 432x288 with 1 Axes\u003e"
            ]
          },
          "metadata": {
            "needs_background": "light",
            "tags": []
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "print('Review length:')\n",
        "train_texts_lengths = [len(x.split(' ')) for x in X_train_texts]\n",
        "print(\"Mean %.2f words (stddev: %f)\" % \\\n",
        "      (np.mean(train_texts_lengths),\n",
        "       np.std(train_texts_lengths)))\n",
        "\n",
        "# plot review lengths\n",
        "plt.boxplot(train_texts_lengths)\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jvNNLiMDFlGM",
        "outputId": "797cfc9f-a85c-4a34-ef18-f8a5f37d5a5a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "70000\n",
            "3000\n",
            "7000\n"
          ]
        }
      ],
      "source": [
        "# Tokenize the texts and keep only the top n words\n",
        "\n",
        "TOP_WORDS = 10000\n",
        "\n",
        "tokenizer = create_tokenizer(X_train_texts, top_words=TOP_WORDS)\n",
        "\n",
        "X_train = tokenizer.texts_to_sequences(X_train_texts)\n",
        "X_val = tokenizer.texts_to_sequences(X_val_texts)\n",
        "X_test = tokenizer.texts_to_sequences(X_test_texts)\n",
        "\n",
        "print(len(X_train))\n",
        "print(len(X_val))\n",
        "print(len(X_test))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uo3z3yvLFlGM"
      },
      "outputs": [],
      "source": [
        "index_to_token = {tokenizer.word_index[k]: k for k in tokenizer.word_index.keys()}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7rYrmNXHFlGM"
      },
      "outputs": [],
      "source": [
        "MAX_WORDS = 500\n",
        "\n",
        "X_train = pad_sequences(X_train, max_words=MAX_WORDS)\n",
        "X_val = pad_sequences(X_val, max_words=MAX_WORDS)\n",
        "X_test = pad_sequences(X_test, max_words=MAX_WORDS)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "I0tt9etTFlGM",
        "outputId": "48a301f5-59b9-420e-d9c1-edf3fe649ed0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding (Embedding)        (None, 500, 100)          1000000   \n",
            "_________________________________________________________________\n",
            "conv1d (Conv1D)              (None, 498, 128)          38528     \n",
            "_________________________________________________________________\n",
            "global_max_pooling1d (Global (None, 128)               0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 1)                 129       \n",
            "=================================================================\n",
            "Total params: 1,038,657\n",
            "Trainable params: 1,038,657\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "tf.reset_default_graph()\n",
        "model = tf.keras.Sequential([\n",
        "    tf.keras.Input(shape=(MAX_WORDS,)),\n",
        "    tf.keras.layers.Embedding(TOP_WORDS, 100),\n",
        "    tf.keras.layers.Conv1D(filters=128, strides=1,\n",
        "                           kernel_size=3,\n",
        "                           activation='relu'),\n",
        "    tf.keras.layers.GlobalMaxPooling1D(),\n",
        "    tf.keras.layers.Dense(1, activation='sigmoid')\n",
        "    ])\n",
        "model.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wbRhbM_RFlGN",
        "outputId": "f4bdcb5a-53ef-436f-8355-a9af94aceae4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train on 70000 samples, validate on 3000 samples\n",
            "Epoch 1/10\n",
            "70000/70000 [==============================] - ETA: 0s - loss: 0.0344 - acc: 0.1689"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/Users/subhamsahoo/Library/Python/3.7/lib/python/site-packages/tensorflow/python/keras/engine/training.py:2325: UserWarning: `Model.state_updates` will be removed in a future version. This property should not be used in TensorFlow 2.0, as `updates` are applied automatically.\n",
            "  warnings.warn('`Model.state_updates` will be removed in a future version. '\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "70000/70000 [==============================] - 57s 816us/sample - loss: 0.0344 - acc: 0.1689 - val_loss: 0.0260 - val_acc: 0.1573\n",
            "Epoch 2/10\n",
            "70000/70000 [==============================] - 60s 859us/sample - loss: 0.0229 - acc: 0.1691 - val_loss: 0.0241 - val_acc: 0.1573\n",
            "Epoch 3/10\n",
            "70000/70000 [==============================] - 65s 924us/sample - loss: 0.0182 - acc: 0.1692 - val_loss: 0.0246 - val_acc: 0.1567\n",
            "Epoch 4/10\n",
            "70000/70000 [==============================] - 77s 1ms/sample - loss: 0.0140 - acc: 0.1693 - val_loss: 0.0253 - val_acc: 0.1567\n",
            "Epoch 5/10\n",
            "70000/70000 [==============================] - 74s 1ms/sample - loss: 0.0103 - acc: 0.1694 - val_loss: 0.0262 - val_acc: 0.1563\n",
            "Epoch 6/10\n",
            "70000/70000 [==============================] - 76s 1ms/sample - loss: 0.0074 - acc: 0.1694 - val_loss: 0.0271 - val_acc: 0.1567\n",
            "Epoch 7/10\n",
            "70000/70000 [==============================] - 73s 1ms/sample - loss: 0.0054 - acc: 0.1694 - val_loss: 0.0284 - val_acc: 0.1563\n",
            "Epoch 8/10\n",
            "70000/70000 [==============================] - 77s 1ms/sample - loss: 0.0041 - acc: 0.1694 - val_loss: 0.0293 - val_acc: 0.1557\n",
            "Epoch 9/10\n",
            "70000/70000 [==============================] - 79s 1ms/sample - loss: 0.0035 - acc: 0.1694 - val_loss: 0.0290 - val_acc: 0.1560\n",
            "Epoch 10/10\n",
            "70000/70000 [==============================] - 71s 1ms/sample - loss: 0.0033 - acc: 0.1694 - val_loss: 0.0301 - val_acc: 0.1553\n"
          ]
        }
      ],
      "source": [
        "model.compile(optimizer=tf.keras.optimizers.Adam(),\n",
        "              loss='mse',  # binary_crossentropy\n",
        "              metrics=['acc'])\n",
        "\n",
        "# multi-input test\n",
        "history = model.fit(X_train, y_train, epochs=10, batch_size=128,\n",
        "                    validation_data=(X_val, y_val))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dR748IVcFlGN"
      },
      "outputs": [],
      "source": [
        "def load_rationale_annotations(path, verbose=True):\n",
        "  data = []\n",
        "  fopen = gfile.Open(path, 'rb')\n",
        "  if path.endswith(\".gz\"):\n",
        "    fopen = gzip.GzipFile(fileobj=fopen)\n",
        "\n",
        "  for line in fopen:\n",
        "    item = json.loads(line)\n",
        "    data.append(item)\n",
        "  fopen.close()\n",
        "  if verbose:\n",
        "    print('Loaded %d annotations.' % len(data))\n",
        "  return data\n",
        "\n",
        "\n",
        "def retokenize_annotation(annotation, tokenizer, annotation_indices=None, count_missing=False):\n",
        "  joined_annotation = ' '.join(annotation)\n",
        "  retokenized = tokenizer.texts_to_sequences([joined_annotation])[0]\n",
        "  if count_missing:\n",
        "    word_sequence = text.text_to_word_sequence(joined_annotation,\n",
        "                                                tokenizer.filters,\n",
        "                                                tokenizer.lower,\n",
        "                                                tokenizer.split)\n",
        "    num_missing = len(word_sequence) - len(retokenized)\n",
        "  else:\n",
        "    num_missing = None\n",
        "  return retokenized, num_missing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xtn7dYxqFlGN",
        "outputId": "cecb631e-394c-4ae2-bb77-a8a1e611554b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded 994 annotations.\n"
          ]
        }
      ],
      "source": [
        "annotations = load_rationale_annotations(\n",
        "    os.path.join(BASE_PATH, 'annotations.json'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "32LFCxVKFlGO",
        "outputId": "2d7d6b2d-96dc-4327-fbc5-71b2a15cc4f2"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEICAYAAABWJCMKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de5gV1Z3v//dHriIIKKgB5CKYCKggtngFRbygZ5QhYgKiBjOGHH8xyZiYGU7M1TNmzGMeh8TjccREExNsJN5nvGvIqCcZpEFEAQ0tttiIiCAKImrD9/dHVdO7293Nbrrp3U19Xs+zn9571aq1v1XdXd9da62qrYjAzMyyZ59iB2BmZsXhBGBmllFOAGZmGeUEYGaWUU4AZmYZ5QRgZpZRTgAGgKR2krZI6t+cdYtJ0hBJe2Sec922JT0hadqeiEPSDyX9++6ub1YfJ4A2Kj0AVz92SPoo53XeA1FDImJ7RHSNiNXNWbe1kvSUpB/lKb9A0hpJ7RrTXkScFRFzmiGuMyRV1Gn7f0fE/2xq27t4z5D03T31HtY6OQG0UekBuGtEdAVWA+fllH3mQCSpfctH2ar9DrgkT/klwB8iYnsLx1NMXwE2Ape29Bv777K4nAD2UpL+RdLdkkolbQYulnSipP+WtEnSWkm/ktQhrd8+/RQ4MH39h3T5o5I2S/qrpEGNrZsuP0fS3yS9L+kmSf9P0vR64i4kxq9LKpf0nqRf5azbTtK/SdogaRUwoYFddB9wiKSTctY/EDgXuDN9fb6kJZI+kLRa0g8b2N/PVW/TruKQdLmkFem+ek3S5Wl5d+A/gP45Z3MHpb/L3+asP0nSsnQf/UnSF3KWVUr6jqSX0v1dKqlTA3F3A74I/H/AMEkj6ywfm/4+3pf0pqRL0vIu6TauTpc9I6lTvjOYNKbT0ueN+rtM1zkqPWPbKOltSf8kqa+krZJ65NQbnS53UilURPjRxh9ABXBGnbJ/AT4BziNJ9PsCxwHHA+2Bw4C/AVem9dsDAQxMX/8BeBcoAToAd5N8Mm5s3YOAzcDEdNl3gE+B6fVsSyExPgh0BwaSfHI9I11+JbAM6AccCDyT/InXu9/uAP495/U3gLKc16cDw9P9NyLdxr9Llw3JbRt4rnqbdhVH+js5DFD6Hh8BR6fLzgAq8vwuf5s+HwpsSdfrAHwfeBXokC6vBP4bOCR9778BlzewDy5L19kHeBT4t5xlg9L3+lK673sBI9NltwJPA58D2gGnpPHki78SOG03/y67A+uAbwOdgP2B0emyJ4Cv5bzPTbnx+1HAsaPYAfjRDL/E+hPAn3ax3tXAH9Pn+Q7quQfH84GXd6PuV4Fnc5YJWEs9CaDAGE/IWX4fcHX6/Jncgx3Jp/looO3TSBJIp/T1AuCbDdT/P8AN6fOGEkBj4/hP4Bvp810lgJ8Cd+Us2wd4GzglfV0JTMlZfiPwfxp47z8Dv0ifX5IebNunr39Yve/rrNMO+BgYnmdZIQmgMX+XlwAL66k3DfivnL+N9cCo5v7/2psf7gLau72Z+0LSEZIeTk+TPwCuJflUV5+3c55vBbruRt0+uXFE8t9aWV8jBcZY0HsBbzQQL8B/AR8A50n6PHAMUJoTy4mS/ixpvaT3gcvzxJJPg3FI+jtJC9IujU3AWQW2W932zvYiYgfJ/uybU6eg31vahTcWqB4zuj+tW91ldSjwWp5VDwY61rOsEI35u6wvhup4RyiZjTYBeCciFu9mTJnkBLB3qzv18FbgZWBIROwP/IjkE/metJakKwQASaL2waqupsS4luSAUa3BaappMrqTZPDzEuCRiHg3p8pc4F7g0IjoDvy6wFjqjUPSvsA9wL8CB0dED5KujOp2dzVd9C1gQE57+5Ds3zUFxFXXpen7PirpbaCc5MD+lXT5m8DgPOutI+nGybfsQ6BLTnztSbqicjXm77K+GIiIrSS/n2kkv7/f56tn9XMCyJZuwPvAh5KGAl9vgff8T2CUpPPSg8G3gd57KMZ5wD+mA4QHAv9cwDp3knx6/CrJzKC6sWyMiG2STgCmNEMcnUgOsuuB7ZL+Dhifs3wd0CsdnK2v7fMlnZYOlH6PZIxlQYGx5bqU5GA7MufxZZIzop4kXXsTlEyNbS+pl6QRkcyQ+i0wS9Ih6aD3yWk8rwDdJJ2dvv4xydhAQxr6nT9EMih+ZTrIvL+k0TnL7yT53f2PNF5rBCeAbPkuyae7zSSfuu7e028YEetIDio3AhtIPs29QNKH3Nwx3kIyMPkSsJDkk/au4isHnic5MD9cZ/EVwL+ms1W+T3LwbVIcEbEJuIqk+2IjMJkkSVYvf5nkU21FOivmoDrxLiPZP7eQJJEJwPkR8WmBsQEg6RSS7qSbI+Lt6kcaVwXw5Yh4nWSw9p/TWBcDR6VNXAWsABaly34GKCLeA75JkkzXpMtyu6Tyqfd3HhHvA2cCF5Akx78Bp+as+wxJ//+CiKi3a9HyUzqAYtYilFxg9RYwOSKeLXY81vZJega4PSJ+W+xY2hqfAdgeJ2mCpB7pfPQfkkwDfb7IYdleIO2aOxL4Y7FjaYucAKwlnAKsIumyOBuYFBH1dQGZFUTSHOAx4NsR8WGx42mLCuoCkjQB+CXJ/N9fR8T1dZaPBWYBR5PMQb4nLR8H/FtO1SPS5Q8ouVJ0LskMgUXAJRHxSdM3yczMCrHLM4C0z/Zm4BxgGDBV0rA61VYD04G7cgsjYn5EjIyIkSRXLm4lmfIG8HOSq/aGAO8B/9CE7TAzs0Yq5J4Zo4HyiFgFIGkuyWX9y6srRERFumxHA+1MBh6NiK3pXPDTgYvSZb8DfkIys6FevXr1ioEDBxYQspmZVVu0aNG7EfGZ6deFJIC+1L5yr5Lkvh2NNYVkKiAk3T6bIqIqp828FwdJmgHMAOjfvz9lZWW78dZmZtklKe9V8S0yCCzpcyTzhx9v7LoRMTsiSiKipHfvhq4fMjOzxigkAayh9mXtu3PZ+ZeA+3MuVtkA9Mi5bevuXspuZma7qZAEsBA4XNIgSR1JunIeauT7TCXnJlvpPVjmk4wLQHIV4IONbNPMzJpglwkg7ae/kqT7ZgUwLyKWSbpW0vkAko6TVAlcCNwqaVn1+ukdBw8lufNirn8GviOpnGRM4DdN3xwzMytUm7oVRElJSXgQ2MyscSQtioiSuuW+Etgya84cGDgQ9tkn+TmnyV/p7jgcRxuLodjfSNOYx7HHHhvW9v3hDxEDBkRIyc8//KE4MXTpEgE1jy5dWj4Wx+E4WiIGcr7qNPdR9IN6Yx5OAE1X7INva/jHiki2PTeG6seAAY7DcbSOOJozhvoSgMcAMmTOHJgxA7ZurSnr0gVmz4Zp01omhoED4Y08l6QMGAAVFS0TAySn1Pn+9CXY0dD17I7DcbRQHM0Zg8cAiqw19Cdec03tgz8kr6+5puViWL26ceV7Sv96viyyvnLH4ThaOo6WiMEJoAVUf/J+440ko7/xRvK6pZNAazj4toZ/LIDrrkvOfnJ16ZKUOw7H0RriaJEY8vULtdZHWx0DaA39ia0ljtYyBlAdS7EHox2H42iJGMjqIHBr+CVK+Q+8UsvG0VoOvq3hd2KWJfUlgELuBtpm1R30rO56gZYb9ISkeyPfwGdLd3tUb/M11yTdPv37J6eTLbkvquNo6fc0s8/aq2cBtZYZJ61h9o2ZZVcmZwG1hkFPSA7ys2cniUdKfvrgb2bFtld3AbWWrhdwt4eZtT579RlAa5jKZWbWWu3VCcBdL2Zm9duru4DAXS9mZvXZq88AzMysfk4AZmYZ5QRgZpZRTgBmZhlVUAKQNEHSq5LKJc3Ms3yspMWSqiRNrrOsv6QnJK2QtDz9kngkjU/XWSLpOUlDmmODzMysMLtMAJLaATcD5wDDgKmShtWpthqYDtyVp4k7gRsiYigwGngnLb8FmBYRI9P1frA7G2BmZrunkGmgo4HyiFgFIGkuMBFYXl0hIirSZbW+pyZNFO0j4sm03pacxQHsnz7vDry1e5tgZma7o5AE0Bd4M+d1JXB8ge1/Htgk6T5gEPAUMDMitgOXA49I+gj4ADghXwOSZgAzAPoX4x4OZmZ7qT09CNweGANcDRwHHEbSVQRwFXBuRPQD7gBuzNdARMyOiJKIKOndu/ceDtfMLDsKSQBrgENzXvdLywpRCSyJiFURUQU8AIyS1BsYEREL0np3AycV2KaZmTWDQhLAQuBwSYMkdQSmAA8V2P5CoEd6wAc4nWTs4D2gu6TPp+VnAisKD9vMzJpql2MAEVEl6UrgcaAdcHtELJN0LcnXjD0k6TjgfqAncJ6kn0bE8IjYLulq4GlJAhYBt6Vtfg24Nx04fg/46h7aRjMzy2Ov/kYwMzPL6DeCmZlZ/ZwAzMwyygnAzCyjnADMzDLKCcDMLKOcAMzMMsoJwMwso5wAzMwyygnAzCyjnADMzDLKCcDMLKOcAMzMMsoJwMwso5wAzMwyygnAzCyjnADMzDLKCcDMLKOcAMzMMsoJwMwsowpKAJImSHpVUrmkmXmWj5W0WFKVpMl1lvWX9ISkFZKWSxqYlkvSdZL+li77VnNskJmZFab9ripIagfcDJwJVAILJT0UEctzqq0GpgNX52niTuC6iHhSUldgR1o+HTgUOCIidkg6aLe3wszMGm2XCQAYDZRHxCoASXOBicDOBBARFemyHbkrShoGtI+IJ9N6W3IWXwFcFBE70mXv7P5mmJlZYxXSBdQXeDPndWVaVojPA5sk3SfpBUk3pGcUAIOBL0sqk/SopMMLD9vMzJpqTw8CtwfGkHQNHQccRtL1A9AJ2BYRJcBtwO35GpA0I00SZevXr9/D4ZqZZUchCWANSV99tX5pWSEqgSURsSoiqoAHgFE5y+5Ln98PHJ2vgYiYHRElEVHSu3fvAt/WzMx2pZAEsBA4XNIgSR2BKcBDBba/EOghqfrIfTo1YwcPAOPS56cCfyuwTTMzawa7TADpJ/crgceBFcC8iFgm6VpJ5wNIOk5SJXAhcKukZem620m6f56W9BIgku4egOuBC9LyfwUub95NMzOzhigiih1DwUpKSqKsrKzYYZiZtSmSFqXjrbX4SmAzs4xyAjAzyygnADOzjHICMDPLKCcAM7OMcgIwM8soJwAzs4xyAjAzyygnADOzjHICMDPLKCcAM7OMcgIwM8soJwAzs4xyAjAzyygnADOzjHICMDPLKCcAM7OMcgIwM8soJwAzs4xyAjAzy6iCEoCkCZJelVQuaWae5WMlLZZUJWlynWX9JT0haYWk5ZIG1ln+K0lbmrIRZmbWeLtMAJLaATcD5wDDgKmShtWpthqYDtyVp4k7gRsiYigwGngnp+0SoOduRW5mZk1SyBnAaKA8IlZFxCfAXGBiboWIqIiIpcCO3PI0UbSPiCfTelsiYmu6rB1wA/BPTd8MMzNrrEISQF/gzZzXlWlZIT4PbJJ0n6QXJN2QHvgBrgQeioi1DTUgaYakMkll69evL/BtzcxsV/b0IHB7YAxwNXAccBgwXVIf4ELgpl01EBGzI6IkIkp69+69R4M1M8uS9gXUWQMcmvO6X1pWiEpgSUSsApD0AHAC8DYwBCiXBNBFUnlEDCk0cDMza5pCEsBC4HBJg0gO/FOAiwpsfyHQQ1LviFgPnA6URcTDwCHVlSRt8cHfzKxl7bILKCKqSPrrHwdWAPMiYpmkayWdDyDpOEmVJN06t0palq67naT752lJLwECbtszm2JmZo2hiCh2DAUrKSmJsrKyYodhZtamSFoUESV1y30lsJlZRhUyBmBmGfbpp59SWVnJtm3bih2K7ULnzp3p168fHTp0KKi+E4CZNaiyspJu3boxcOBA0ll71gpFBBs2bKCyspJBgwYVtI67gMysQdu2bePAAw/0wb+Vk8SBBx7YqDM1JwAz2yUf/NuGxv6enADMrFXbsGEDI0eOZOTIkRxyyCH07dt35+tPPvmkoDYuu+wyXn311Qbr3HzzzcyZM6c5QgZg3bp1tG/fnl//+tfN1mZz8zRQM2vQihUrGDp0aMH158yBa66B1auhf3+47jqYNq15YvnJT35C165dufrqq2uVRwQRwT77tJ7PtDfddBPz5s2jY8eOPP300y32vvl+X54GamZ73Jw5MGMGvPEGRCQ/Z8xIyptbeXk5w4YNY9q0aQwfPpy1a9cyY8YMSkpKGD58ONdee+3OuqeccgpLliyhqqqKHj16MHPmTEaMGMGJJ57IO+8kd6j/wQ9+wKxZs3bWnzlzJqNHj+YLX/gCf/nLXwD48MMPueCCCxg2bBiTJ0+mpKSEJUuW5I2vtLSUWbNmsWrVKtaurbnn5cMPP8yoUaMYMWIEZ511FgCbN2/mK1/5CkcffTRHH300DzzwQPPvsDycAMys2VxzDWzdWrts69akfE945ZVXuOqqq1i+fDl9+/bl+uuvp6ysjBdffJEnn3yS5cuXf2ad999/n1NPPZUXX3yRE088kdtvvz1v2xHB888/zw033LAzmdx0000ccsghLF++nB/+8Ie88MILedetqKhg48aNHHvssVx44YXMmzcPgLfffpsrrriC+++/nxdffJG5c+cCyZlN7969Wbp0KS+++CKnnnpqc+yeXXICMLNms3p148qbavDgwZSU1PRslJaWMmrUKEaNGsWKFSvyJoB9992Xc845B4Bjjz2WioqKvG1/8Ytf/Eyd5557jilTpgAwYsQIhg8fnnfduXPn8uUvfxmAKVOmUFpaCsBf//pXxo0bx4ABAwA44IADAHjqqaf4xje+ASQDuT17tsz3ZPk6ADNrNv37J90++cr3hP3222/n85UrV/LLX/6S559/nh49enDxxRfnnRLZsWPHnc/btWtHVVVV3rY7deq0yzr1KS0t5d133+V3v/sdAG+99RarVq1qVBstwWcAZtZsrrsOunSpXdalS1K+p33wwQd069aN/fffn7Vr1/L44483+3ucfPLJO7tzXnrppbxnGMuXL6eqqoo1a9ZQUVFBRUUF3/ve95g7dy4nnXQS8+fP5400S27cuBGAM888k5tvvhlIup7ee++9Zo89HycAM2s206bB7NkwYABIyc/Zs5tvFlBDRo0axbBhwzjiiCO49NJLOfnkk5v9Pb75zW+yZs0ahg0bxk9/+lOGDRtG9+7da9UpLS1l0qRJtcouuOACSktLOfjgg7nllluYOHEiI0aMYFq6Y3784x+zbt06jjzySEaOHMmzzz7b7LHn42mgZtagxk4D3ZtVVVVRVVVF586dWblyJWeddRYrV66kffvW05vemGmgrSdqM7NWbsuWLYwfP56qqioigltvvbVVHfwbq+1GbmbWwnr06MGiRYuKHUaz8RiAmVlGOQGYmWWUE4CZWUYVlAAkTZD0qqRySTPzLB8rabGkKkmT6yzrL+kJSSskLZc0MC2fk7b5sqTbJRX2FTZmZtYsdpkAJLUDbgbOAYYBUyUNq1NtNTAduCtPE3cCN0TEUGA08E5aPgc4AjgK2Be4fDfiN7O93Lhx4z5zUdesWbO44oorGlyva9euQHIV7uTJk/PWOe2009jV1PJZs2axNecGR+eeey6bNm0qJPSCjBw5cuftJVpaIWcAo4HyiFgVEZ8Ac4GJuRUioiIilgI7csvTRNE+Ip5M622JiK3p80ciBTwP9Gv65pjZ3mbq1Kk7b5pWbe7cuUydOrWg9fv06cM999yz2+9fNwE88sgj9OjRY7fby7VixQq2b9/Os88+y4cfftgsbTZGIQmgL/BmzuvKtKwQnwc2SbpP0guSbkjPKHZKu34uAR7L14CkGZLKJJWtX7++wLc1s73F5MmTefjhh3d++UtFRQVvvfUWY8aM2Tkvf9SoURx11FE8+OCDn1m/oqKCI488EoCPPvqIKVOmMHToUCZNmsRHH320s94VV1yx81bSP/7xjwH41a9+xVtvvcW4ceMYN24cAAMHDuTdd98F4MYbb+TII4/kyCOP3Hkr6YqKCoYOHcrXvvY1hg8fzllnnVXrfXKVlpZyySWXcNZZZ9WKvby8nDPOOIMRI0YwatQoXnvtNQB+/vOfc9RRRzFixAhmzvxMb3yj7enrANoDY4BjSLqJ7ibpKvpNTp3/CzwTEXmvfY6I2cBsSK4E3pPBmlnD/vEfoZ7b3++2kSMhPXbmdcABBzB69GgeffRRJk6cyNy5c/nSl76EJDp37sz999/P/vvvz7vvvssJJ5zA+eefX+9XI95yyy106dKFFStWsHTpUkaNGrVz2XXXXccBBxzA9u3bGT9+PEuXLuVb3/oWN954I/Pnz6dXr1612lq0aBF33HEHCxYsICI4/vjjOfXUU+nZsycrV66ktLSU2267jS996Uvce++9XHzxxZ+J5+677+bJJ5/klVde4aabbuKiiy4CYNq0acycOZNJkyaxbds2duzYwaOPPsqDDz7IggUL6NKly877CDVFIWcAa4BDc173S8sKUQksSbuPqoAHgJ17XNKPgd7Adwpsz8wyKLcbKLf7JyL4/ve/z9FHH80ZZ5zBmjVrWLduXb3tPPPMMzsPxNVfvlJt3rx5jBo1imOOOYZly5blvdFbrueee45Jkyax33770bVrV774xS/uvIfPoEGDGDlyJFD/LafLysro1asX/fv3Z/z48bzwwgts3LiRzZs3s2bNmp33E+rcuTNdunThqaee4rLLLqNLere96ltJN0UhZwALgcMlDSI58E8BLiqw/YVAD0m9I2I9cDpQBiDpcuBsYHxE7GigDTNrJRr6pL4nTZw4kauuuorFixezdetWjj32WADmzJnD+vXrWbRoER06dGDgwIF5bwG9K6+//jq/+MUvWLhwIT179mT69Om71U616ltJQ3I76XxdQKWlpbzyyisMHDgQSO5meu+997bogPAuzwDST+5XAo8DK4B5EbFM0rWSzgeQdJykSuBC4FZJy9J1twNXA09LegkQcFva9L8DBwN/lbRE0o+aedvMbC/RtWtXxo0bx1e/+tVag7/vv/8+Bx10EB06dKh1m+X6jB07lrvuSiYrvvzyyyxduhRIDr777bcf3bt3Z926dTz66KM71+nWrRubN2/+TFtjxozhgQceYOvWrXz44Yfcf//9jBkzpqDt2bFjB/PmzeOll17aecvoBx98kNLSUrp160a/fv12fi3kxx9/zNatWznzzDO54447dg5IN0cXUEFjABHxCPBInbIf5TxfSD2zeNIZQEfnKfd9iMysYFOnTmXSpEm1ZgRNmzaN8847j6OOOoqSkhKOOOKIBtu44ooruOyyyxg6dChDhw7deSYxYsQIjjnmGI444ggOPfTQWreSnjFjBhMmTKBPnz7Mnz9/Z/moUaOYPn06o0ePBuDyyy/nmGOOqfcbxnI9++yz9O3blz59+uwsGzt2LMuXL2ft2rX8/ve/5+tf/zo/+tGP6NChA3/84x+ZMGECS5YsoaSkhI4dO3Luuefys5/9rKB9Vx/fDtrMGuTbQbctjbkdtG8FYWaWUU4AZmYZ5QRgZpZRTgBmtkttaawwyxr7e3ICMLMGde7cmQ0bNjgJtHIRwYYNG+jcuXPB63gqppk1qF+/flRWVuJ7cbV+nTt3pl+/wu+r6QRgZg3q0KEDgwYNKnYYtge4C8jMLKOcAMzMMsoJwMwso5wAzMwyygnAzCyjnADMzDLKCcDMLKOcAMzMMsoJwMwso5wAzMwyygnAzCyjCkoAkiZIelVSuaSZeZaPlbRYUpWkyXWW9Zf0hKQVkpZLGpiWD5K0IG3zbkkdm2ODzMysMLtMAJLaATcD5wDDgKmShtWpthqYDtyVp4k7gRsiYigwGngnLf858G8RMQR4D/iH3dkAMzPbPYWcAYwGyiNiVUR8AswFJuZWiIiKiFgK7MgtTxNF+4h4Mq23JSK2ShJwOnBPWvV3wN83bVPMzKwxCkkAfYE3c15XpmWF+DywSdJ9kl6QdEN6RnEgsCkiqnbVpqQZksoklfl+5GZmzWdPDwK3B8YAVwPHAYeRdBUVLCJmR0RJRJT07t27+SM0M8uoQhLAGuDQnNf90rJCVAJL0u6jKuABYBSwAeghqfoLaRrTppmZNYNCEsBC4PB01k5HYArwUIHtLyQ50Fd/dD8dWB7Jl4vOB6pnDH0FeLDwsM3MrKl2mQDST+5XAo8DK4B5EbFM0rWSzgeQdJykSuBC4FZJy9J1t5N0/zwt6SVAwG1p0/8MfEdSOcmYwG+ad9PMzKwhSj6Mtw0lJSVRVlZW7DDMzNoUSYsioqRuua8ENjPLKCcAM7OMcgIwM8soJwAzs4xyAjAzyygnADOzjHICMDPLKCcAM7OMcgIwM8soJwAzs4xyAjAzyygnADOzjHICMDPLKCcAM7OMcgIwM8soJwAzs4xyAjAzyygnADOzjHICMDPLqIISgKQJkl6VVC5pZp7lYyUtllQlaXKdZdslLUkfD+WUj0/XWSLpOUlDmr45ZmZWqF0mAEntgJuBc4BhwFRJw+pUWw1MB+7K08RHETEyfZyfU34LMC0iRqbr/WA34jczs93UvoA6o4HyiFgFIGkuMBFYXl0hIirSZTsa8d4B7J8+7w681Yh1zcysiQpJAH2BN3NeVwLHN+I9OksqA6qA6yPigbT8cuARSR8BHwAn5FtZ0gxgBkD//v0b8bZmZtaQlhgEHhARJcBFwCxJg9Pyq4BzI6IfcAdwY76VI2J2RJREREnv3r1bIFwzs2woJAGsAQ7Ned0vLStIRKxJf64C/gwcI6k3MCIiFqTV7gZOKrRNMzNrukISwELgcEmDJHUEpgAP7WIdACT1lNQpfd4LOJlk7OA9oLukz6dVzwRWNDZ4MzPbfbscA4iIKklXAo8D7YDbI2KZpGuBsoh4SNJxwP1AT+A8ST+NiOHAUODWdHB4H5IxgOUAkr4G3Jsuew/46p7YQDMzy08RUewYClZSUhJlZWXFDsPMrE2RtCgdi63FVwKbmWWUE4CZWUYVch1Am/fpp9Baero6dACp2FGYmWUkAfz938MjjxQ7ikT//nD22TBhAowfD927FzsiM8uqTCSASy+Fk08udhSwfTssXgxz58Jtt0G7dnDCCUlCOPtsOPbYpMzMrCV4FlARfDhGuqcAAAnFSURBVPop/Pd/w+OPJ49Fi5IuqgMOgDPPrEkIffoUO1Iz2xvUNwvICaAVWL8ennoKHnsMnngC3n47KT/qqJpkcMop0LlzceM0s7bJCaCNiIClS2vODp57Dj75BPbdF047rSYhfOELHkw2s8I4AbRRH34If/5zkgweewxWrkzKBwyoSQYeTDazhjgB7CVef73m7ODpp2HzZg8mm1nDnAD2Qh5MNrNCOAFkgAeTzSwfJ4CMaWgw+dRT4aCDih1h0k3Vvz8MGQKDByc/e/XKxuB2VRWsXg3l5fDaa8nPbdvgsMNq9sdhh0GXLsWO1PYGTgAZlzuYPH8+bNlS7IiShLR2be3bdOy/f00yqPuzTx/Ypw3dvWrbtmTMJvcgX/28oiJJAtX23Rc6dYJNm2q30bdv/fvDA/9WKCcAa5U+/jg5SNY9QJaXJ+W5B8nOnWt/Qs79OWAAtC/Cde2bN9fEW3cbKis/m9yGDKl55G7D5z6XJLeNGz/bXvXP6i69ar161Z8cevfOxpmUFcYJwNqcqip48838yeG11+Cjj2rqtmsHAwfmPxgOGpR8wt4dEbBhQ/0xvPNO7foHHVTzvnVjOfDAph2Ut2yBVavyJ4fVq2snm27d6k8Offu2rTMpazonANurRCTdR/kOzOXl8P77tev361f/gblr1/rbeu212m1JSVv5DqyDBycH3mL4+OOkWylfcnj99WTGWLVOnRo+k+rQoTjbYHuOE4BlRkT+rpTq5+vW1a7foUPtA2T79g2fTbS1WVTbtydnUvn2RXn5Z8+kcmeNnXwydOxYvNiteTgBmKU2b67dlfLuu8mBvfog379/ccYTiiEiGVuo3hcrVyYzxv7yl6QLbr/9YNy45PblZ5+d7B9re5qUACRNAH5J8qXwv46I6+ssHwvMAo4GpkTEPTnLtgMvpS9XR8T5abmAfwEuBLYDt0TErxqKwwnArGV88EEyW6x6GvGqVUn5YYfVnB2cfnrxuryscXY7AUhqB/wNOBOoBBYCUyNieU6dgcD+wNXAQ3USwJaI6Jqn3cuAccD0iNgh6aCIeKduvVxOAGbFUV5ecz+q+fOTacXt28NJJ9WcHYwc6cHl1qopXwo/GiiPiFUR8QkwF5iYWyEiKiJiKbCjETFdAVwbETvSNho8+JtZ8QwZAt/4BvzHfySzov70J/jud5Mzhe9/P7n/1CGHwMUXw+9//9lxlrYmItmGxYth69ZiR7PnFNLT2Rd4M+d1JXB8I96js6QyoAq4PiIeSMsHA1+WNAlYD3wrIlY2ol0zK4JOnZJxgXHj4PrrkzGEJ56o6S6aMyepN3JkzdnBSSe1vsHkHTuSazXqm/1VfbFkp04wZkxN19eRR+4911i0xFDXgIhYI+kw4E+SXoqI14BOwLaIKJH0ReB2YEzdlSXNAGYA9O/fvwXCNbPGOOSQ5GtXL700Oai+8EJNMvjFL5Ik0bVrkjCqD6ItNZj86acNT4/9+OOauh06JGMcgwcnt0sZPDjZtgULkq6v730vefTpA2edlSS3M85Iru9oqwoZAzgR+ElEnJ2+/l8AEfGveer+FvjP3DGA+pZLegU4JyJeTweEN0VEgxe3ewzArG2pHkx+7LEkIbz+elJePZg8YUKSGJoymLx1a8MXyG3fXlO3S5f803uHDEmu72joNupvvllzpvPUU/Dee8mZwHHH1SS2449vnTPImjII3J5kEHg8sIZkEPiiiFiWp+5vyUkAknoCWyPiY0m9gL8CEyNiuaTrgb9FxO2STgNuiIjjGorFCcCs7YqoGUyuvidV9WDyySfXHETzDSZv2lT/LTLeeqt23Z49P3twr35+8MHN032zfTssXFizLQsWJGc/3bsnX9BUvS0DBjT9vZpDU6eBnksyzbMdcHtEXCfpWqAsIh6SdBxwP9AT2Aa8HRHDJZ0E3EoyOLwPMCsifpO22QOYA/QHtgD/MyJebCgOJwCzvcfHHyfXG1TPLnox/e8/6KCkawVqDvIbNtRe95BD8l/VPXhw8n0YLe2995KzguqEUFmZlB9xRE0yOPXU4t3d1ReCmVmrljuY/Kc/JVdc57vdxmGHJWMKrVUErFhRkwz+67+SO8MWczDZCcDMrAg++gieeaYmISxPr6Dq06cmGezpwWQnADOzVqAYg8lOAGZmrUz1YHL1LKnnn68ZTD7jjJqE0NQZ8E4AZmat3MaN8PTT+QeT77kHhg/fvXbrSwCtcMaqmVk2HXAAXHhh8sgdTH7yyT0zpdQJwMysFZJg2LDkcdVVe+Y9fO8+M7OMcgIwM8soJwAzs4xyAjAzyygnADOzjHICMDPLKCcAM7OMcgIwM8uoNnUrCEnrgTeKHUcT9QLeLXYQrYT3RW3eH7V5f9Ro6r4YEBG96xa2qQSwN5BUlu+eHFnkfVGb90dt3h819tS+cBeQmVlGOQGYmWWUE0DLm13sAFoR74vavD9q8/6osUf2hccAzMwyymcAZmYZ5QRgZpZRTgAtQNKhkuZLWi5pmaRvFzum1kBSO0kvSPrPYsdSbJJ6SLpH0iuSVkg6sdgxFYukq9L/k5cllUrqXOyYWpKk2yW9I+nlnLIDJD0paWX6s2dzvJcTQMuoAr4bEcOAE4BvSBpW5Jhag28DK4odRCvxS+CxiDgCGEFG94ukvsC3gJKIOBJoB0wpblQt7rfAhDplM4GnI+Jw4On0dZM5AbSAiFgbEYvT55tJ/rn7Fjeq4pLUD/gfwK+LHUuxSeoOjAV+AxARn0TEpuJGVVTtgX0ltQe6AG8VOZ4WFRHPABvrFE8Efpc+/x3w983xXk4ALUzSQOAYYEFxIym6WcA/ATuKHUgrMAhYD9yRdon9WtJ+xQ6qGCJiDfALYDWwFng/Ip4oblStwsERsTZ9/jZwcHM06gTQgiR1Be4F/jEiPih2PMUi6e+AdyJiUbFjaSXaA6OAWyLiGOBDmukUv61J+7YnkiTFPsB+ki4ublStSyRz95tl/r4TQAuR1IHk4D8nIu4rdjxFdjJwvqQKYC5wuqQ/FDekoqoEKiOi+qzwHpKEkEVnAK9HxPqI+BS4DzipyDG1BuskfQ4g/flOczTqBNACJImkf3dFRNxY7HiKLSL+V0T0i4iBJAN8f4qIzH7Ki4i3gTclfSEtGg8sL2JIxbQaOEFSl/T/ZjwZHRCv4yHgK+nzrwAPNkejTgAt42TgEpJPukvSx7nFDspalW8CcyQtBUYCPytyPEWRngXdAywGXiI5RmXqlhCSSoG/Al+QVCnpH4DrgTMlrSQ5S7q+Wd7Lt4IwM8smnwGYmWWUE4CZWUY5AZiZZZQTgJlZRjkBmJlllBOAmVlGOQGYmWXU/w8S5ru++NeK+AAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "\u003cFigure size 432x288 with 1 Axes\u003e"
            ]
          },
          "metadata": {
            "needs_background": "light",
            "tags": []
          },
          "output_type": "display_data"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEICAYAAABWJCMKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de3gV1b3/8feXa0AQFLDchIBYIZGLGLEW0XKsCi0aL1ShEdHqj9ZitVX7k9baR/1Jq6c+1drj0VLFqo1SD9bKqVrbKoqXVgnINahEDBBABRQEgkLg+/tjTchOCMkOSfZOmM/refaz96xZM3vN5Mn6zqxZey1zd0REJH5apLsAIiKSHgoAIiIxpQAgIhJTCgAiIjGlACAiElMKACIiMaUAIA3CzFqa2XYz69OQedPJzAaYWaP0k666bzP7u5nlNUY5zOxmM3vgYLevYb9XmtnLDb1fSR0FgJiKKuDy114z25mwXG1FVBN33+PuHdx9TUPmbarM7J9m9vNq0i80s3Vm1rIu+3P3s9w9vwHK9XUzK66y7//n7t+r777l0KMAEFNRBdzB3TsAa4BzEtL2q4jMrFXqS9mkPQJMqiZ9EvBHd9+T4vKI1JkCgFTLzG43sz+Z2RNmtg24xMxOMbN/m9kWM9tgZveaWesofyszczPLjJb/GK1/3sy2mdm/zKxfXfNG68ea2XtmttXMfmtmr5vZZQcodzJl/K6ZFZnZp2Z2b8K2Lc3sbjPbbGargDE1nKI/A93N7KsJ23cBvgE8Gi2fa2aLzOwzM1tjZjfXcL5fKz+m2soRNb2siM7V+2Z2ZZTeCfhfoE/C3dxR0d/yDwnbn29my6Nz9JKZHZewrsTMrjOzpdH5fsLM2tZwHhLLdaqZFUTbvWVmJyesu8LMiqMyrzKzCVH6l81sXrTNJjN7PJnvkgbi7nrF/AUUA1+vknY7sAs4h3Ch0A44CTgZaAX0B94Dro7ytwIcyIyW/whsAnKA1sCfCFfGdc17FLANyI3WXQfsBi47wLEkU8ZngE5AJvBJ+bEDVwPLgd5AF2Be+Bc54Hl7GHggYXkqUJCw/B9AdnT+hkbHOC5aNyBx38Br5cdUWzmiv0l/wKLv2AkMidZ9HSiu5m/5h+jzIGB7tF1r4KfAu0DraH0J8G+ge/Td7wFXHuD4rwRejj53BbYCE6PzPAnYDBwBHB6tOzbK2wPIij7/D3BjdI4ygJHp/n+I00t3AFKT19z9f919r7vvdPf57v6mu5e5+ypgBnB6DdvPdvcCd98N5APDDiLvOGCRuz8TrbubUJFWK8ky/tLdt7p7MfBywnddBNzt7iXuvhm4o4byQmgGuijhCvnSKK28LC+5+/Lo/C0GZlVTlurUWI7ob7LKg5eAF4FRSewXYAIwJyrb7mjfnQhBs9w97v5h9N1/pea/W7lzgOXu/kR07h8DVgHfLC82cLyZZbj7BncvjNJ3EwJxD3f/3N1fT/I4pAEoAEhN1iYumNlAM3vWzD40s8+A2whXfgfyYcLnUqDDQeTtmVgOd3fCVWq1kixjUt8FrK6hvACvAJ8B55jZl4ETgCcSynKKmb1sZhvNbCvhirmm81WuxnKY2Tgze9PMPjGzLcBZSe63fN/79ufuewnns1dCnrr83ardb0K5e7n7Z4Q7g6nAh2b21+h8AVxPuBMpiJqdJid5HNIAFACkJlW7Hv4OWAYMcPfDgZ8TmiEa0wZCUwgAZmZUrqyqqk8ZNwBHJyzX2E01CkaPEq78JwHPuXvi3cks4CngaHfvBDyYZFkOWA4zawfMBn4JfMndOwN/T9hvbd1F1wN9E/bXgnB+1yVRrqT3G+lTvl93f97dv05o/iki/J2I7gaudPcehAAxI/H5jzQuBQCpi46EttwdZjYI+G4KvvOvwHAzO8dCT6RrgW6NVMYngR+aWa/oge6NSWzzKOEh7XdIaP5JKMsn7v65mX2F0PxS33K0BdoAG4E9ZjYOOCNh/UdAVzPrWMO+zzWzr0UPx39MeMbyZpJlO5C/AtlmdnH0sP3bhOccz5pZj+jv157wXGkHsBfAzC4ys/KAvoUQwNSDKkUUAKQurgcmEyqM3xEe1jYqd/8IuBj4NeGh4jHA28AXjVDG+wnt6UuB+YQr7drKVwS8RaiYn62y+irglxZ6Uf2UUPnWqxzuvgX4EfA04QH2eELlW75+GeGuozjq5XNUlfIuJ5yf+wlBZAxwbvQ84KC5+0bgXEKw2hyVcZy7fwq0JASaDdG6rxKu9iE8e5hvZjsIPaumejP+fUhzY+EuVqR5sPADq/XAeHd/Nd3lEWnOdAcgTZ6ZjTGzzlFvm5sJPUfeSnOxRJo9BQBpDk4ldCncCJwNnO/uB2oCEpEkqQlIRCSmdAcgIhJTzWqAr65du3pmZma6iyEi0qwsWLBgk7vv1326WQWAzMxMCgoK0l0MEZFmxcyq/VW7moBERGJKAUBEJKaSCgBRP+x3ozHUp1Wzvq2FseOLokGqMqP0EdF46IvMbLGZnZ+wTXE0+NMiM1O7johIitX6DCD65eV9wJmEUQPnm9mchOFcAa4APnX3AdFED3cSfr6/DMhx9zIz6wEsNrP/dfeyaLvRVQbPEhGRFEnmDmAEUBSNP76LMMJhbpU8uVQMhDUbOMPMzN1LEyr7DGofqVBERFIkmQDQi8pjk1cdO7xSnqjC30qYTQgzO9nMlhMGtvpeQkBw4O9mtsDMphzoy81sSjTNXMHGjRuTOaZK8vMhMxNatAjv+fWedltE5NDQ6A+Bo9mZsglT9f3EzDKiVae6+3BgLDDVzE47wPYz3D3H3XO6datpFOD95efDlCmwejW4h/cpUxQEREQguQCwjsqTU1Q3ecS+PNGY7Z0Iw77u4+4rCHORHh8tl08U8TFhaNsRdS9+zW66CUpLK6eVloZ0EZG4SyYAzAeONbN+ZtaGaE7RKnnmEMYYhzA++Uvu7tE2rQDMrC8wkDBO+WHlE1aY2WGEKe2W1f9wKltzgFHFD5QuIhIntfYCinrwXA28QJjYYaa7Lzez24ACd58DPAQ8ZmZFhEkqymc+OhWYZma7CTMAfd/dN5lZf+DpMLsfrYDH3f1vDX1wffqEZp/q0kVE4q5ZjQaak5PjdRkKovwZQGIzUPv2MGMG5OU1QgFFRJogM1vg7jlV0w/pXwLn5YXKvm9fMAvvqvxFRIJmNRjcwcjLU4UvIlKdQ/oOQEREDkwBQEQkphQARERiSgFARCSmFABERGJKAUBEJKYUAEREYkoBQEQkphQARERiSgFARCSmFABERGJKAUBEJKYUAEREYkoBQEQkphQARERiSgFARCSmFABERGJKAUBEJKYUAEREYkoBQEQkppIKAGY2xszeNbMiM5tWzfq2ZvanaP2bZpYZpY8ws0XRa7GZnZ/sPkVEpHHVGgDMrCVwHzAWyAImmllWlWxXAJ+6+wDgbuDOKH0ZkOPuw4AxwO/MrFWS+xQRib09e6CoqHH23SqJPCOAIndfBWBms4BcoDAhTy5wS/R5NvBfZmbuXpqQJwPwOuxTRCRW3KGkBN56q+JVUAA7d8K2bdCuXcN+XzIBoBewNmG5BDj5QHncvczMtgJdgE1mdjIwE+gLTIrWJ7NPAMxsCjAFoE+fPkkUV0SkediyJVTw5ZX9m2/Chx+GdW3awLBhcNllMGJECA4NLZkAUC/u/iaQbWaDgEfM7Pk6bj8DmAGQk5PTCKdARKTxffEFLF5c+er+3Xcr1h93HJx1VqjsR4yAIUOgbdvGLVMyAWAdcHTCcu8orbo8JWbWCugEbE7M4O4rzGw7cHyS+xQRaZb27oWVKytf2S9aBLt3h/Xdu8PJJ8Oll4bKPicHOndOfTmTCQDzgWPNrB+hkp4AfLtKnjnAZOBfwHjgJXf3aJu1UbNPX2AgUAxsSWKfIiLNwoYNla/s58+HrVvDug4dQgX/ox+FSn/ECOjVC8zSW2ZIIgBElffVwAtAS2Cmuy83s9uAAnefAzwEPGZmRcAnhAod4FRgmpntBvYC33f3TQDV7bOBj01EmgF3+OgjWLEC3nknvK9cCS1bwuGHQ8eONb9XTWvVyA3b27bBggXhqr68wi8pCetatQpNNxMnVjTlDBwYjqUpMm+MJwuNJCcnxwsKCtJdDBE5CHv2wAcfVFTyiRX+li0V+Tp0gC9/OQSGbdvgs8/C+86dyX1Pu3a1B41kg8revbB0aeWr+8LCigeyxxxTUdGffHJ4aNvQPXUagpktcPecqumN/hBYROJl5054772KSr68on/vvfAgtFz37uHqeOJEGDQovAYOPHDzyO7dsH17RUCo6b1qWklJ5eXPP0/uWFq0CEEAoGvXUMlfdFGo8E86Cbp0qf/5SicFABE5KJs3V381X1xccYXcogX07x8q9jFjKir5gQPhiCPq9n2tW4dt6rpddXbvrj2IbNsGu3bB4MGhws/MbBrt9g1JAUBEDsgd1q7dv5JfsQI2bqzIl5ERujGefHLotz5wYKjsjz02rGtqWreGI48MrzhTABARAD75BF5/HZYsqajk330XduyoyHPkkaFiz82tuJofNAj69g1X+9K8KACIxNTHH8O8efDKK+F96dKKpps+fULFftppFZX8oEGhHfxQawaJMwUAkZhYt66iwn/lldCcA9C+PYwcGR5unnYanHBC6Ikjhz4FAJFDVHFxxdX9K6/A+++H9MMPh1NPhcsvh9NPh+HDQ5u4xI8CgMghwD0MGVx+dT9vHqxZE9YdeSSMGgVTp4YKf+jQpvvDJEktBQCRZsg9PKRNrPA3bAjrjjoqVPQ//nF4z87WA1qpngKASDOwd2/onZP40HbTprCuVy8YPTpU9qedFrpj6kGtJEMBQKQJKiuDt9+uqOxffbViuITMTPjmN0OFf/rp0K+fKnw5OAoAIk3AF1+EAcbKm3Refz0MewBhXJzx4yuu8DUvkjQUBQCRFNq9O4yJs3x55dfKlWGwNAht9pdeGir7006DHj3SW2Y5dCkAiDSCsrLQ7XLZssoV/XvvVUwK0qJFGE0yOztc4Z9wQuit061bessu8aEAIFIP5UMcl1fw5RX+O++EgcQgtM/36xcq+nPOgeOPD5+PO65pDh0s8aEAIJKEvXth9erKV/PLloWumIlDC/ftGyr3s88O79nZYQiFww5LX9lFDkQBQCRB+eiXVdvoCwsrD4rWq1eo3EePrqjos7LCJCIizYUCgMSSexgbp7Bw/4r+s88q8nXvHir3K66oaLrJykrPBN4iDS0WAWDdutCTQr+GjJ+9e8OYOIWFobmmsLDi87ZtFfm6dg0V/KRJFVf02dnNf8YnkZrEIgCMGxcmr/j2tyEvL0zarB/OHFrKe92UV/Dllfw771SeS7ZHj9AmP3lyuJIfNCi8H3VU+souki6HfABwhxtvhPx8uPtu+NWvwpVdXl4ICH37pruEUhdffBG6Ula9ok/sXgnhx1JZWaGNvrySHzSoYaYTFDlUmJfPANEM5OTkeEFBwUFvv2kTPPlkCAZvvBHSRo0KweBb39L0cE3Jjh0V0w8mXtEXFVVM0m0W+tGXV/Dlr4EDNZ69SCIzW+DuOfulxykAJPrgA3j88RAMVqwI46GPHRuCwTnnqH92qmzdun8lX1gY2u3LtWoV5pZNrOQHDQpDJOjvJFK7egUAMxsD/AZoCTzo7ndUWd8WeBQ4EdgMXOzuxWZ2JnAH0AbYBfzY3V+KtnkZ6AGUt9Ce5e4f11SOhgwA5dxh0SL44x/hiSfCkLodO8KFF4ZgMHq0xk6vD3fYvDm0z69aVfG+alUY/mD9+oq8bduGq/fEtvmsLBgwQBOWiNTHQQcAM2sJvAecCZQA84GJ7l6YkOf7wBB3/56ZTQDOd/eLzewE4CN3X29mxwMvuHuvaJuXgRvcPekavTECQKI9e+Dll8NdwVNPhe6APXrAhAkhGAwfrofH1dm9O0w+kljBJ1b0id0qIZzT/v1D803iFX2/fgq2Io2hPgHgFOAWdz87Wv4JgLv/MiHPC1Gef5lZK+BDoJsn7NzMjHB30MPdv2iKASDRzp3w7LMhGDz7bKjkjjsOLrkkPDzu3z8lxWgytm6tvoJ///1Q+ZcPZAbhSr5fv4pKPvG9X78wB62IpM6BAkAyvYB6AWsTlkuAkw+Ux93LzGwr0AXYlJDnQmChu3+RkPawme0BngJu92qikZlNAaYA9EnhOLjt2oUBusaPh08+gdmzQzC4+ebwOuWUcFdw0UWHxuBde/aE30scqJL/5JPK+bt2DZX6V74SzkP//hUVfc+e+s2FSHOQzB3AeGCMu18ZLU8CTnb3qxPyLIvylETL70d5NkXL2cAcQjv/+1FaL3dfZ2YdCQHgj+7+aE1lSeUdwIGsWROeFeTnw9Kl4QHlWWeFSjA3t+mO+bJ7N3z4YXjGsX59eAieWMEXF1cMXgbhuPr2rbhyT7yK798/TCwuIs1Dfe4A1gFHJyz3jtKqy1MSNQF1IjT3YGa9gaeBS8srfwB3Xxe9bzOzx4ERhAfJTVqfPuF3BTfeGKboy88PvYmeey5U/uefH4LB178eKtHGtmNHRcVe02vTpv237dQpVOpDhoRyJ1byRx+dmvKLSPok8y8+HzjWzPoRKvoJwLer5JkDTAb+BYwHXnJ3N7POwLPANHd/vTxzFCQ6u/smM2sNjAP+We+jSbEhQ8Lrl78MU/bl58P//E/oUXTUUXDxxeGZwUkn1e3hsXuY/q+2Sn3DhsrDGZRr3TqMYVP+sHXkyPA58dWvX/hRlB5qi8RXst1AvwHcQ+gGOtPdp5vZbUCBu88xswzgMeAE4BNggruvMrOfAT8BVibs7ixgBzAPaB3t85/Ade6e8Chxf02hCag2X3wR7gby8+Gvfw3LAwaEu4KJE8NVd3nlXdOV+xdf7L/v9u33r8irex15pNrgRaSCfgiWBlu2wJ//HILB3Lnhyr46RxyRXMWuoYZF5GDU5xmAHKTOneE73wmvdevgL38J6YmVevfukJGR3nKKSDwpAKRIr14wdWq6SyEiUkEtxSIiMaUAICISUwoAKZKfD5mZoXdOZmZYFhFJJz0DSIH8fJgyBUpLw/Lq1WEZQvdQEZF00B1ACtx0U0XlX660NKSLiKSLAkAKrFlTt3QRkVRQAEiBAw1imsLBTUVE9qMAkALTp+8/Bn779iFdRCRdFABSIC8PZswIwyubhfcZM/QAWETSS72AUiQvTxW+iDQtugMQEYkpBQARkZhSABARiSkFABGRmFIAEBGJKQUAEZGYUgAQEYkpBQARkZhSABARiSkFABGRmFIAEBGJqaQCgJmNMbN3zazIzKZVs76tmf0pWv+mmWVG6Wea2QIzWxq9/0fCNidG6UVmdq+ZWUMdlIiI1K7WAGBmLYH7gLFAFjDRzLKqZLsC+NTdBwB3A3dG6ZuAc9x9MDAZeCxhm/uB/wMcG73G1OM4RESkjpK5AxgBFLn7KnffBcwCcqvkyQUeiT7PBs4wM3P3t919fZS+HGgX3S30AA5393+7uwOPAufV+2hERCRpyQSAXsDahOWSKK3aPO5eBmwFulTJcyGw0N2/iPKX1LJPAMxsipkVmFnBxo0bkyiuiIgkIyUPgc0sm9As9N26buvuM9w9x91zunXr1vCFExGJqWQCwDrg6ITl3lFatXnMrBXQCdgcLfcGngYudff3E/L3rmWfIiLSiJIJAPOBY82sn5m1ASYAc6rkmUN4yAswHnjJ3d3MOgPPAtPc/fXyzO6+AfjMzL4S9f65FHimnsciIiJ1UGsAiNr0rwZeAFYAT7r7cjO7zczOjbI9BHQxsyLgOqC8q+jVwADg52a2KHodFa37PvAgUAS8DzzfUAclIiK1s9AJp3nIycnxgoKCdBdDRKRZMbMF7p5TNV2/BBYRiSkFABGRmFIAiJn8fMjMhBYtwnt+frpLJCLp0irdBZDUyc+HKVOgtDQsr14dlgHy8tJXLhFJD90BxMhNN1VU/uVKS0O6iMSPAkCMrFlTt3QRObQpAMRInz51SxeRQ5sCQIxMnw7t21dOa98+pItI/CgAxEheHsyYAX37gll4nzFDD4BF4kq9gGImL08VvogEugMQEYkpBQARkZhSABARiSkFABGRmFIAEBGJKQUAEZGYUgAQEYkpBQARkZhSABARiSkFABGRmFIAEBGJKQUAEZGYSioAmNkYM3vXzIrMbFo169ua2Z+i9W+aWWaU3sXM5prZdjP7ryrbvBztc1H0OqohDkhERJJT62igZtYSuA84EygB5pvZHHcvTMh2BfCpuw8wswnAncDFwOfAzcDx0auqPHcvqOcxiIjIQUjmDmAEUOTuq9x9FzALyK2SJxd4JPo8GzjDzMzdd7j7a4RAICIiTUgyAaAXsDZhuSRKqzaPu5cBW4EuSez74aj552YzsyTyi4hIA0nnQ+A8dx8MjIpek6rLZGZTzKzAzAo2btyY0gKKiBzKkgkA64CjE5Z7R2nV5jGzVkAnYHNNO3X3ddH7NuBxQlNTdflmuHuOu+d069YtieKKiEgykgkA84FjzayfmbUBJgBzquSZA0yOPo8HXnJ3P9AOzayVmXWNPrcGxgHL6lp4ERE5eLUGgKhN/2rgBWAF8KS7Lzez28zs3CjbQ0AXMysCrgP2dRU1s2Lg18BlZlZiZllAW+AFM1sCLCLcQfy+4Q5LmrL8fMjMhBYtwnt+frpLJBJPVsOFepOTk5PjBQXqNdqc5efDlClQWlqR1r49zJihyepFGouZLXD3nKrp+iWwpNRNN1Wu/CEs33RTesojEmcKAJJSa9bULV1EGo8CgKRUnz51SxeRxqMAICk1fXpo80/Uvn1IF5HUUgCQlMrLCw98+/YFs/CuB8Ai6VHrYHAiDS0vTxW+SFOgOwARkZhSABARiSkFABGRmFIAEBGJKQUAEZGYUgAQEYkpBQARkZhSABARiSkFABGRmFIAEBGJKQUAEZGYUgAQEYkpBQARkZhSABARiSkFABGRmFIAkNjKz4fMTGjRIrzn56e7RCKppQlhJJby82HKFCgtDcurV4dl0GQ1Eh9J3QGY2Rgze9fMisxsWjXr25rZn6L1b5pZZpTexczmmtl2M/uvKtucaGZLo23uNTNriAMSScZNN1VU/uVKS0O6SFzUGgDMrCVwHzAWyAImmllWlWxXAJ+6+wDgbuDOKP1z4Gbghmp2fT/wf4Bjo9eYgzkAkYOxZk3d0kUORcncAYwAitx9lbvvAmYBuVXy5AKPRJ9nA2eYmbn7Dnd/jRAI9jGzHsDh7v5vd3fgUeC8+hyISF306VO3dJFDUTIBoBewNmG5JEqrNo+7lwFbgS617LOkln0CYGZTzKzAzAo2btyYRHFFajd9OrRvXzmtffuQLhIXTb4XkLvPcPccd8/p1q1buosjh4i8PJgxA/r2BbPwPmOGHgBLvCTTC2gdcHTCcu8orbo8JWbWCugEbK5ln71r2adIo8rLU4Uv8ZbMHcB84Fgz62dmbYAJwJwqeeYAk6PP44GXorb9arn7BuAzM/tK1PvnUuCZOpdeREQOWq13AO5eZmZXAy8ALYGZ7r7czG4DCtx9DvAQ8JiZFQGfEIIEAGZWDBwOtDGz84Cz3L0Q+D7wB6Ad8Hz0EhGRFLEaLtSbnJycHC8oKEh3MUREmhUzW+DuOVXTm/xDYBERaRwKACIiMaUAICISUwoAIiIxpQAgIhJTCgAiIjGlACAiElMKACIiMaUAIJJmmppS0qXZTwm5e/duSkpK+Pzzz2vPLGmXkZFB7969ad26dbqL0iRoakpJp2Y/FMQHH3xAx44d6dKlC5pVsmlzdzZv3sy2bdvo169fuovTJGRmhkq/qr59obg41aWRQ9UhOxTE559/rsq/mTAzunTporu1BJqaUtKp2QcAQJV/M6K/VWWamlLS6ZAIACLNlaamlHSKXQBo6B4XmzdvZtiwYQwbNozu3bvTq1evfcu7du1Kah+XX3457777bo157rvvPvIbqHvIqaeeyqJFixpkX1I/mppS0qnZ9wKqi8bocdGlS5d9lektt9xChw4duOGGGyrlcXfcnRYtqo+3Dz/8cK3fM3Xq1IMroDR5mppS0iVWdwA33VRR+ZcrLQ3pDa2oqIisrCzy8vLIzs5mw4YNTJkyhZycHLKzs7ntttv25S2/Ii8rK6Nz585MmzaNoUOHcsopp/Dxxx8D8LOf/Yx77rlnX/5p06YxYsQIjjvuON544w0AduzYwYUXXkhWVhbjx48nJycn6Sv9nTt3MnnyZAYPHszw4cOZN28eAEuXLuWkk05i2LBhDBkyhFWrVrFt2zbGjh3L0KFDOf7445k9e3ZDnjoRSZFYBYBU97h45513+NGPfkRhYSG9evXijjvuoKCggMWLF/OPf/yDwsLC/bbZunUrp59+OosXL+aUU05h5syZ1e7b3Xnrrbf41a9+tS+Y/Pa3v6V79+4UFhZy88038/bbbydd1nvvvZe2bduydOlSHnvsMSZNmsSuXbv47//+b2644QYWLVrE/Pnz6dmzJ8899xyZmZksXryYZcuWceaZZx7cCRKRtIpVAEh1j4tjjjmGnJyKrrdPPPEEw4cPZ/jw4axYsaLaANCuXTvGjh0LwIknnkjxATqDX3DBBfvlee2115gwIUzHPHToULKzs5Mu62uvvcYll1wCQHZ2Nj179qSoqIivfvWr3H777fznf/4na9euJSMjgyFDhvC3v/2NadOm8frrr9OpU6ekv0dEmo5YBYBU97g47LDD9n1euXIlv/nNb3jppZdYsmQJY8aMqbY/fJs2bfZ9btmyJWVlZdXuu23btrXmaQiTJk3i6aefpm3btowZM4Z58+YxaNAgCgoKyM7OZtq0afziF79otO8XkcYTqwCQzh4Xn332GR07duTwww9nw4YNvPDCCw3+HSNHjuTJJ58EQtt9dXcYBzJq1Kh9vYxWrFjBhg0bGDBgAKtWrWLAgAFce+21jBs3jiVLlrBu3To6dOjApEmTuP7661m4cGGDH4uINL5Y9QKC9PW4GD58OFlZWQwcOJC+ffsycuTIBv+OH/zgB1x66aVkZWXtex2oeebss8/eNx7PqFGjmDlzJt/97ncZPHgwrVu35tFHH6VNmyDINCYAAAsOSURBVDY8/vjjPPHEE7Ru3ZqePXtyyy238MYbbzBt2jRatGhBmzZteOCBBxr8WCT18vNDh4g1a0Kz6PTp6p10qGv2YwGtWLGCQYMGpalETUtZWRllZWVkZGSwcuVKzjrrLFauXEmrVk0rzutv1vRU7SINoXlUv0k4NNRrLCAzG2Nm75pZkZlNq2Z9WzP7U7T+TTPLTFj3kyj9XTM7OyG92MyWmtkiMyuouk+pu+3btzNy5EiGDh3KhRdeyO9+97smV/lL05TKLtLSdNRaO5hZS+A+4EygBJhvZnPcPbGB+QrgU3cfYGYTgDuBi80sC5gAZAM9gX+a2ZfdfU+03Wh339SAxxNrnTt3ZsGCBekuhjRDGpQunpK5AxgBFLn7KnffBcwCcqvkyQUeiT7PBs6wMOpXLjDL3b9w9w+Aomh/ItKEaFC6eEomAPQC1iYsl0Rp1eZx9zJgK9Cllm0d+LuZLTCzKQf6cjObYmYFZlawcePGJIorInWlQeniKZ3dQE919+HAWGCqmZ1WXSZ3n+HuOe6e061bt9SWUCQmNChdPCUTANYBRycs947Sqs1jZq2ATsDmmrZ19/L3j4GnUdOQSFrl5YVZyPbuDe/pqvw1R3LqJBMA5gPHmlk/M2tDeKg7p0qeOcDk6PN44CUP/UvnABOiXkL9gGOBt8zsMDPrCGBmhwFnAcvqfzipN3r06P1+1HXPPfdw1VVX1bhdhw4dAFi/fj3jx4+vNs/XvvY1qnZ7reqee+6hNKH7xje+8Q22bNmSTNFrdMstt3DXXXfVez8idVHeHXX1anCvGLFXQaBx1BoAojb9q4EXgBXAk+6+3MxuM7Nzo2wPAV3MrAi4DpgWbbsceBIoBP4GTI16AH0JeM3MFgNvAc+6+98a9tBSY+LEicyaNatS2qxZs5g4cWJS2/fs2bNeo2lWDQDPPfccnTt3Puj9iaSTuqOmVlKdxN39OeC5Kmk/T/j8OfCtA2w7HZheJW0VMLSuha3ND38IDT3PybBhEI3CXK3x48fzs5/9jF27dtGmTRuKi4tZv349o0aNYvv27eTm5vLpp5+ye/dubr/9dnJzK3egKi4uZty4cSxbtoydO3dy+eWXs3jxYgYOHMjOnTv35bvqqquYP38+O3fuZPz48dx6663ce++9rF+/ntGjR9O1a1fmzp1LZmYmBQUFdO3alV//+tf7RhO98sor+eEPf0hxcTFjx47l1FNP5Y033qBXr14888wztGvX7oDHuGjRIr73ve9RWlrKMcccw8yZMzniiCO49957eeCBB2jVqhVZWVnMmjWLV155hWuvvRYI0z/OmzePjh071uMvIHGi7qipFauxgBrDkUceyYgRI3j++eeBcPV/0UUXYWZkZGTw9NNPs3DhQubOncv1119PTb+8vv/++2nfvj0rVqzg1ltvrdSnf/r06RQUFLBkyRJeeeUVlixZwjXXXEPPnj2ZO3cuc+fOrbSvBQsW8PDDD/Pmm2/y73//m9///vf7hodeuXIlU6dOZfny5XTu3JmnnnqqxmO89NJLufPOO1myZAmDBw/m1ltvBeCOO+7g7bffZsmSJfuGg7jrrru47777WLRoEa+++mqNgUWkqqbSHTUuzyEOqZ+J1nSl3pjKm4Fyc3OZNWsWDz30EBDG7P/pT3/KvHnzaNGiBevWreOjjz6ie/fu1e5n3rx5XHPNNQAMGTKEIUOG7Fv35JNPMmPGDMrKytiwYQOFhYWV1lf12muvcf755+8bkfSCCy7g1Vdf5dxzz6Vfv34MGzYMqHnIaQjzE2zZsoXTTz8dgMmTJ/Otb31rXxnz8vI477zzOO+884AwIN11111HXl4eF1xwAb17907mFIoAodtpdUNSpLI7amPMHNhU6Q6gAeTm5vLiiy+ycOFCSktLOfHEEwHIz89n48aNLFiwgEWLFvGlL32p2iGga/PBBx9w11138eKLL7JkyRK++c1vHtR+ypUPJQ31G0762WefZerUqSxcuJCTTjqJsrIypk2bxoMPPsjOnTsZOXIk77zzzkGXU+KnKXRHbUrPIRr7TkQBoAF06NCB0aNH853vfKfSw9+tW7dy1FFH0bp1a+bOncvq1atr3M9pp53G448/DsCyZctYsmQJEIaSPuyww+jUqRMfffTRvuYmgI4dO7Jt27b99jVq1Cj+8pe/UFpayo4dO3j66acZNWpUnY+tU6dOHHHEEbz66qsAPPbYY5x++uns3buXtWvXMnr0aO688062bt3K9u3bef/99xk8eDA33ngjJ510kgKA1Fm6u6M2lecQqegRdUg1AaXTxIkTOf/88yv1CMrLy+Occ85h8ODB5OTkMHDgwBr3cdVVV3H55ZczaNAgBg0atO9OYujQoZxwwgkMHDiQo48+utJQ0lOmTGHMmDH7ngWUGz58OJdddhkjRoSfV1x55ZWccMIJNTb3HMgjjzyy7yFw//79efjhh9mzZw+XXHIJW7duxd255ppr6Ny5MzfffDNz586lRYsWZGdn75vdTKS56NMnVLbVpadSTXciDRUUNRy0pJz+ZtKUNZWhsVu0CFf+VZmFu6O6qNdw0CIicdEUnkNAanpEKQCIiFSR7ucQkJoB+g6JANCcmrHiTn8rkeSk4k6k2T8EzsjIYPPmzXTp0oUwBYE0Ve7O5s2bycjISHdRRJqFxp7DvNkHgN69e1NSUoLmCmgeMjIy9OMwkSai2QeA1q1b069fv3QXQ0Sk2TkkngGIiEjdKQCIiMSUAoCISEw1q18Cm9lGoOYBdZq+rsCmdBeiidC5qEznozKdjwr1PRd93X2/SdWbVQA4FJhZQXU/yY4jnYvKdD4q0/mo0FjnQk1AIiIxpQAgIhJTCgCpNyPdBWhCdC4q0/moTOejQqOcCz0DEBGJKd0BiIjElAKAiEhMKQCkgJkdbWZzzazQzJab2bXpLlNTYGYtzextM/trusuSbmbW2cxmm9k7ZrbCzE5Jd5nSxcx+FP2fLDOzJ8wsVsPHmtlMM/vYzJYlpB1pZv8ws5XR+xEN8V0KAKlRBlzv7lnAV4CpZpaV5jI1BdcCK9JdiCbiN8Df3H0gMJSYnhcz6wVcA+S4+/FAS2BCekuVcn8AxlRJmwa86O7HAi9Gy/WmAJAC7r7B3RdGn7cR/rl7pbdU6WVmvYFvAg+muyzpZmadgNOAhwDcfZe7b0lvqdKqFdDOzFoB7YH1aS5PSrn7POCTKsm5wCPR50eA8xriuxQAUszMMoETgDfTW5K0uwf4v0Adp7c+JPUDNgIPR01iD5rZYekuVDq4+zrgLmANsAHY6u5/T2+pmoQvufuG6POHwJcaYqcKAClkZh2Ap4Afuvtn6S5PupjZOOBjd1+Q7rI0Ea2A4cD97n4CsIMGusVvbqK27VxCUOwJHGZml6S3VE2Lh777DdJ/XwEgRcysNaHyz3f3P6e7PGk2EjjXzIqBWcB/mNkf01uktCoBSty9/K5wNiEgxNHXgQ/cfaO77wb+DHw1zWVqCj4ysx4A0fvHDbFTBYAUsDBZ8UPACnf/dbrLk27u/hN37+3umYQHfC+5e2yv8tz9Q2CtmR0XJZ0BFKaxSOm0BviKmbWP/m/OIKYPxKuYA0yOPk8GnmmInSoApMZIYBLhSndR9PpGugslTcoPgHwzWwIMA36R5vKkRXQXNBtYCCwl1FGxGhLCzJ4A/gUcZ2YlZnYFcAdwppmtJNwl3dEg36WhIERE4kl3ACIiMaUAICISUwoAIiIxpQAgIhJTCgAiIjGlACAiElMKACIiMfX/AQzFG+W/1k6AAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "\u003cFigure size 432x288 with 1 Axes\u003e"
            ]
          },
          "metadata": {
            "needs_background": "light",
            "tags": []
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "# plotting the results\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "acc = history.history.get('acc')\n",
        "val_acc = history.history.get('val_acc')\n",
        "loss = history.history.get('loss')\n",
        "val_loss = history.history.get('val_loss')\n",
        "\n",
        "epochs = range(1, len(acc) + 1)\n",
        "plt.plot(epochs, acc, 'bo', label='Training Acc')\n",
        "plt.plot(epochs, val_acc, 'b', label='Validation Acc')\n",
        "plt.title('Training and Validation Accuracy')\n",
        "plt.legend()\n",
        "\n",
        "plt.figure()\n",
        "\n",
        "plt.plot(epochs, loss, 'bo', label=\"Training Loss\")\n",
        "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n",
        "plt.title('Training and Validation loss')\n",
        "plt.legend()\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MvNi6R-jFlGO",
        "outputId": "033e2338-e322-4dd8-bc11-a1228d8209b1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets added to graph.\n",
            "INFO:tensorflow:No assets to write.\n",
            "INFO:tensorflow:SavedModel written to: trained_text_model/saved_model.pb\n"
          ]
        }
      ],
      "source": [
        "tf.saved_model.simple_save(\n",
        "    session=tf.keras.backend.get_session(),\n",
        "    export_dir='trained_text_model',\n",
        "    inputs={'input': model.inputs[0]},\n",
        "    outputs={'output': model.outputs[0]})"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3wo7rSkWFlGP"
      },
      "source": [
        "### Generate Saliency Maps"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TCJ21QFTFlGP"
      },
      "outputs": [],
      "source": [
        "def pad_sequences(X, max_words=500):\n",
        "    return keras.preprocessing.sequence.pad_sequences(X, maxlen=max_words)\n",
        "\n",
        "def get_words(x, tokenizer):\n",
        "  index_to_token = {\n",
        "      tokenizer.word_index[k]: k for k in tokenizer.word_index.keys()}\n",
        "  index_to_token[0] = ''\n",
        "  words = []\n",
        "  for i in x:\n",
        "    if i != 0:\n",
        "      words.append(index_to_token[i])\n",
        "  return words\n",
        "\n",
        "\n",
        "def highlight_ground_truth(review, limits):\n",
        "  for lims in limits:\n",
        "    for index in range(lims[0], lims[1], 1):\n",
        "      review[index] = '\u003cu\u003e' + review[index] + '\u003c/u\u003e'\n",
        "  return review\n",
        "\n",
        "\n",
        "def highlight_words(review, words, attributes):\n",
        "  assert len(attributes) == len(words)\n",
        "  previous_index = 0\n",
        "  for word, attribute in zip(words, attributes):\n",
        "    if word not in review:\n",
        "      continue\n",
        "    index = review[previous_index:].index(word)\n",
        "    r, g, b = _get_color(attribute)\n",
        "    review[previous_index + index] = (\n",
        "        \"\u003cspan style='color:rgb(%d,%d,%d)'\u003e%s\u003c/span\u003e\" % (\n",
        "            r, g, b, review[index]))\n",
        "    # previous_index = index\n",
        "  return review\n",
        "\n",
        "def _get_color(attribute):\n",
        "  if attribute \u003e 0:\n",
        "    return (128 - int(64*attribute),\n",
        "            int(128*attribute) + 127,\n",
        "            128 - int(64*attribute),)\n",
        "  return (int(-128*attribute) + 127,\n",
        "          128 + int(64*attribute),\n",
        "          128 + int(64*attribute))\n",
        "\n",
        "\n",
        "def highlight_text(annotation, aspect, words, attributes, print_html=False):\n",
        "  # Color words\n",
        "  review = highlight_words(\n",
        "      review=list(annotation['x']),\n",
        "      words=words,\n",
        "      attributes=attributes)\n",
        "\n",
        "  # Underline words\n",
        "  review = highlight_ground_truth(\n",
        "      review=review,\n",
        "      limits=annotation[str(aspect)])\n",
        "\n",
        "  # New for neurips - remove solid white\n",
        "  for i, word in enumerate(review):\n",
        "    if \"\u003cspan style='color:rgb\" not in word:\n",
        "      review[i] = \"\u003cspan style='color:rgb(128,127,128)'\u003e{}\u003c/span\u003e\".format(word)\n",
        "  display(HTML(' '.join(review)))\n",
        "  \n",
        "  if print_html:\n",
        "    print(' '.join(review))\n",
        "\n",
        "\n",
        "def get_words_with_attributes(x, attributes, tokenizer):\n",
        "  index_to_token = {\n",
        "      tokenizer.word_index[k]: k for k in tokenizer.word_index.keys()}\n",
        "  index_to_token[0] = ''\n",
        "  words = []\n",
        "  _attributes = []\n",
        "  max_attribute = np.percentile(attributes, 99)\n",
        "  if max_attribute == 0:\n",
        "    max_attribute = np.max(attributes)\n",
        "  for i, j in zip(x, attributes):\n",
        "    if i != 0:\n",
        "      words.append(index_to_token[i])\n",
        "      _attributes.append(j /max_attribute)\n",
        "      # print(np.max(np.abs(attributes)), j, np.percentile(attributes, 99))\n",
        "  return words, _attributes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HK0g2yrQFlGQ"
      },
      "outputs": [],
      "source": [
        "run_params = masking.RunParams(**{\n",
        "  'model_path': 'trained_text_model',\n",
        "  'image_placeholder_shape': (1, 500),\n",
        "  'model_type': 'text_cnn',\n",
        "  'padding': (0, 0),\n",
        "  'strides': 1,\n",
        "  'activations': None,\n",
        "  'pixel_range': None,\n",
        "  'tensor_names': {\n",
        "        'input': 'input_1:0',\n",
        "        'embedding': 'embedding/embedding_lookup/Identity_1:0',\n",
        "        'first_layer': 'conv1d/BiasAdd:0',\n",
        "        'first_layer_relu': 'conv1d/Relu:0',\n",
        "        'logits': 'dense/BiasAdd:0',\n",
        "        'softmax': 'dense/Sigmoid:0',\n",
        "        'weights_layer_1': 'conv1d/conv1d/ExpandDims_1:0',\n",
        "        'biases_layer_1': 'conv1d/BiasAdd/ReadVariableOp:0'\n",
        "  }\n",
        "})"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "b2PNh-O3FlGQ",
        "outputId": "a9f4b0d3-b110-4027-a7de-97a6e31f8ce9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /Users/subhamsahoo/smug_colab_test/google-research/smug_saliency/utils.py:189: load (from tensorflow.python.saved_model.loader_impl) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.loader.load or tf.compat.v1.saved_model.load. There will be a new function for importing SavedModels in Tensorflow 2.0.\n",
            "INFO:tensorflow:Restoring parameters from trained_text_model/variables/variables\n",
            "Model type: trained_text_model\n",
            "input (1, 500)\n",
            "embedding (1, 500, 100)\n",
            "first_layer (1, 498, 128)\n",
            "first_layer_relu (1, 498, 128)\n",
            "logits (1, 1)\n",
            "softmax (1, 1)\n",
            "weights_layer_1 (1, 3, 100, 128)\n",
            "biases_layer_1 (128,)\n",
            "\n",
            "\n"
          ]
        }
      ],
      "source": [
        "with utils.restore_model(run_params.model_path) as sess:\n",
        "  run_output = sess.run(\n",
        "      run_params.tensor_names,\n",
        "      feed_dict={run_params.tensor_names['input']: np.random.rand(1, 500)\n",
        "      })\n",
        "  print('Model type: {}'.format(run_params.model_path.split('/')[-1]))\n",
        "  for key, value in run_output.items():\n",
        "    print(key, value.shape)\n",
        "  print('\\n')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "W1WmIX_eFlGQ"
      },
      "outputs": [],
      "source": [
        "annotation_x = pad_sequences([retokenize_annotation(\n",
        "    annotation['x'], tokenizer)[0] for annotation in annotations])\n",
        "\n",
        "annotation_y = np.array(\n",
        "    [float(annotation['y'][ASPECT]) for annotation in annotations])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4ruBr-cUFlGR"
      },
      "outputs": [],
      "source": [
        "def generate_saliency_maps(features, run_params):\n",
        "  smug_result = masking.find_mask_first_layer(\n",
        "    image=features,\n",
        "    label_index=0, # Since only one output node\n",
        "    run_params=run_params,\n",
        "    window_size=1,\n",
        "    score_method='integrated_gradients',\n",
        "    top_k=100,\n",
        "    gamma=0,\n",
        "    timeout=600,\n",
        "    num_unique_solutions=1,\n",
        "  )\n",
        "  smug_base_saliency = masking.get_no_minimization_mask(\n",
        "    image=features,\n",
        "    label_index=0,\n",
        "    top_k=100,\n",
        "    run_params=run_params)\n",
        "  with utils.restore_model(run_params.model_path) as session:\n",
        "    unmasked_predictions = session.run(\n",
        "        run_params.tensor_names,\n",
        "        feed_dict={run_params.tensor_names['input']: [features]})\n",
        "    ig_saliency_map = np.sum(masking.get_saliency_map(\n",
        "        session=session,\n",
        "        features=unmasked_predictions['embedding'][0],\n",
        "        saliency_method='integrated_gradients',\n",
        "        input_tensor_name=run_params.tensor_names['embedding'],\n",
        "        output_tensor_name=run_params.tensor_names['softmax'],\n",
        "        label=0), axis=1)\n",
        "  return (smug_result['masks'][0] * smug_base_saliency,\n",
        "          smug_base_saliency, ig_saliency_map)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dvQ9XTTfFlGR",
        "outputId": "0582c74b-bb79-4459-836c-69f50a92e3a6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Restoring parameters from trained_text_model/variables/variables\n",
            "INFO:tensorflow:Restoring parameters from trained_text_model/variables/variables\n",
            "INFO:tensorflow:Restoring parameters from trained_text_model/variables/variables\n",
            "Smug\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\u003cspan style='color:rgb(127,128,128)'\u003ei\u003c/span\u003e \u003cspan style='color:rgb(93,197,93)'\u003ereally\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003edid\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003enot\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003elike\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ethis\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e.\u003c/span\u003e \u003cspan style='color:rgb(48,288,48)'\u003eit\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ejust\u003c/span\u003e \u003cspan style='color:rgb(79,226,79)'\u003eseemed\u003c/span\u003e \u003cspan style='color:rgb(98,187,98)'\u003eextremely\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ewatery\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e.\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ei\u003c/span\u003e \u003cspan style='color:rgb(32,319,32)'\u003edont\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003e'\u003c/span\u003e \u003cspan style='color:rgb(84,216,84)'\u003ethink\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ethis\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ehad\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003eany\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ecarbonation\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ewhatsoever\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e.\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003emaybe\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003eit\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ewas\u003c/span\u003e \u003cspan style='color:rgb(53,278,53)'\u003eflat\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e,\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ewho\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003eknows\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e?\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ebut\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003eeven\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003eif\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ei\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003egot\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ea\u003c/span\u003e \u003cspan style='color:rgb(-10,404,-10)'\u003ebad\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ebrew\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ei\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003edo\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003en't\u003c/span\u003e \u003cspan style='color:rgb(82,220,82)'\u003esee\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ehow\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ethis\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ewould\u003c/span\u003e \u003cspan style='color:rgb(115,153,115)'\u003epossibly\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ebe\u003c/span\u003e \u003cspan style='color:rgb(115,153,115)'\u003esomething\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ei\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003e'd\u003c/span\u003e \u003cspan style='color:rgb(68,247,68)'\u003eget\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003etime\u003c/span\u003e \u003cspan style='color:rgb(102,179,102)'\u003eand\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003etime\u003c/span\u003e \u003cspan style='color:rgb(103,177,103)'\u003eagain\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e.\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ei\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ecould\u003c/span\u003e \u003cspan style='color:rgb(70,243,70)'\u003etaste\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ethe\u003c/span\u003e \u003cspan style='color:rgb(54,276,54)'\u003ehops\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003etowards\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ethe\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003emiddle\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e,\u003c/span\u003e \u003cspan style='color:rgb(69,246,69)'\u003ebut\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003ethe\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ebeer\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003egot\u003c/span\u003e \u003cspan style='color:rgb(65,254,65)'\u003epretty\u003c/span\u003e \u003cspan style='color:rgb(65,254,65)'\u003enasty\u003c/span\u003e \u003cspan style='color:rgb(89,205,89)'\u003etowards\u003c/span\u003e \u003cspan style='color:rgb(84,216,84)'\u003ethe\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ebottom\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e.\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ei\u003c/span\u003e \u003cspan style='color:rgb(76,232,76)'\u003ewould\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003enever\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003edrink\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ethis\u003c/span\u003e \u003cspan style='color:rgb(65,254,65)'\u003eagain\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e,\u003c/span\u003e \u003cspan style='color:rgb(65,254,65)'\u003eunless\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003eit\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ewas\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003efree\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e.\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003ei\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003e'm\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ekind\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003eof\u003c/span\u003e \u003cspan style='color:rgb(72,239,72)'\u003eupset\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ei\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003ebought\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ethis\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e.\u003c/span\u003e"
            ],
            "text/plain": [
              "\u003cIPython.core.display.HTML object\u003e"
            ]
          },
          "metadata": {
            "tags": []
          },
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "No_minimization\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\u003cspan style='color:rgb(105,173,105)'\u003ei\u003c/span\u003e \u003cspan style='color:rgb(105,173,105)'\u003ereally\u003c/span\u003e \u003cspan style='color:rgb(105,173,105)'\u003edid\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003enot\u003c/span\u003e \u003cspan style='color:rgb(75,234,75)'\u003elike\u003c/span\u003e \u003cspan style='color:rgb(75,234,75)'\u003ethis\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e.\u003c/span\u003e \u003cspan style='color:rgb(75,234,75)'\u003eit\u003c/span\u003e \u003cspan style='color:rgb(95,193,95)'\u003ejust\u003c/span\u003e \u003cspan style='color:rgb(95,193,95)'\u003eseemed\u003c/span\u003e \u003cspan style='color:rgb(108,167,108)'\u003eextremely\u003c/span\u003e \u003cspan style='color:rgb(64,255,64)'\u003ewatery\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e.\u003c/span\u003e \u003cspan style='color:rgb(64,255,64)'\u003ei\u003c/span\u003e \u003cspan style='color:rgb(64,255,64)'\u003edont\u003c/span\u003e \u003cspan style='color:rgb(99,186,99)'\u003e'\u003c/span\u003e \u003cspan style='color:rgb(99,186,99)'\u003ethink\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003ethis\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ehad\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003eany\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003ecarbonation\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003ewhatsoever\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e.\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003emaybe\u003c/span\u003e \u003cspan style='color:rgb(92,200,92)'\u003eit\u003c/span\u003e \u003cspan style='color:rgb(78,227,78)'\u003ewas\u003c/span\u003e \u003cspan style='color:rgb(78,227,78)'\u003eflat\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e,\u003c/span\u003e \u003cspan style='color:rgb(78,227,78)'\u003ewho\u003c/span\u003e \u003cspan style='color:rgb(107,169,107)'\u003eknows\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e?\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ebut\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003eeven\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003eif\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ei\u003c/span\u003e \u003cspan style='color:rgb(92,199,92)'\u003egot\u003c/span\u003e \u003cspan style='color:rgb(86,211,86)'\u003ea\u003c/span\u003e \u003cspan style='color:rgb(36,311,36)'\u003ebad\u003c/span\u003e \u003cspan style='color:rgb(36,311,36)'\u003ebrew\u003c/span\u003e \u003cspan style='color:rgb(36,311,36)'\u003ei\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003edo\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003en't\u003c/span\u003e \u003cspan style='color:rgb(97,189,97)'\u003esee\u003c/span\u003e \u003cspan style='color:rgb(97,189,97)'\u003ehow\u003c/span\u003e \u003cspan style='color:rgb(97,189,97)'\u003ethis\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003ewould\u003c/span\u003e \u003cspan style='color:rgb(120,144,120)'\u003epossibly\u003c/span\u003e \u003cspan style='color:rgb(120,144,120)'\u003ebe\u003c/span\u003e \u003cspan style='color:rgb(120,144,120)'\u003esomething\u003c/span\u003e \u003cspan style='color:rgb(122,139,122)'\u003ei\u003c/span\u003e \u003cspan style='color:rgb(88,207,88)'\u003e'd\u003c/span\u003e \u003cspan style='color:rgb(88,207,88)'\u003eget\u003c/span\u003e \u003cspan style='color:rgb(88,207,88)'\u003etime\u003c/span\u003e \u003cspan style='color:rgb(111,162,111)'\u003eand\u003c/span\u003e \u003cspan style='color:rgb(111,162,111)'\u003etime\u003c/span\u003e \u003cspan style='color:rgb(112,160,112)'\u003eagain\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e.\u003c/span\u003e \u003cspan style='color:rgb(105,174,105)'\u003ei\u003c/span\u003e \u003cspan style='color:rgb(105,174,105)'\u003ecould\u003c/span\u003e \u003cspan style='color:rgb(90,204,90)'\u003etaste\u003c/span\u003e \u003cspan style='color:rgb(88,207,88)'\u003ethe\u003c/span\u003e \u003cspan style='color:rgb(79,226,79)'\u003ehops\u003c/span\u003e \u003cspan style='color:rgb(79,226,79)'\u003etowards\u003c/span\u003e \u003cspan style='color:rgb(79,226,79)'\u003ethe\u003c/span\u003e \u003cspan style='color:rgb(89,206,89)'\u003emiddle\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e,\u003c/span\u003e \u003cspan style='color:rgb(89,206,89)'\u003ebut\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003ethe\u003c/span\u003e \u003cspan style='color:rgb(107,170,107)'\u003ebeer\u003c/span\u003e \u003cspan style='color:rgb(86,211,86)'\u003egot\u003c/span\u003e \u003cspan style='color:rgb(86,211,86)'\u003epretty\u003c/span\u003e \u003cspan style='color:rgb(86,211,86)'\u003enasty\u003c/span\u003e \u003cspan style='color:rgb(102,179,102)'\u003etowards\u003c/span\u003e \u003cspan style='color:rgb(99,186,99)'\u003ethe\u003c/span\u003e \u003cspan style='color:rgb(93,197,93)'\u003ebottom\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e.\u003c/span\u003e \u003cspan style='color:rgb(93,197,93)'\u003ei\u003c/span\u003e \u003cspan style='color:rgb(93,197,93)'\u003ewould\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003enever\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003edrink\u003c/span\u003e \u003cspan style='color:rgb(86,212,86)'\u003ethis\u003c/span\u003e \u003cspan style='color:rgb(86,212,86)'\u003eagain\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e,\u003c/span\u003e \u003cspan style='color:rgb(86,212,86)'\u003eunless\u003c/span\u003e \u003cspan style='color:rgb(89,206,89)'\u003eit\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003ewas\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003efree\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e.\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003ei\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003e'm\u003c/span\u003e \u003cspan style='color:rgb(91,201,91)'\u003ekind\u003c/span\u003e \u003cspan style='color:rgb(91,201,91)'\u003eof\u003c/span\u003e \u003cspan style='color:rgb(91,201,91)'\u003eupset\u003c/span\u003e \u003cspan style='color:rgb(124,136,124)'\u003ei\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003ebought\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003ethis\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e.\u003c/span\u003e"
            ],
            "text/plain": [
              "\u003cIPython.core.display.HTML object\u003e"
            ]
          },
          "metadata": {
            "tags": []
          },
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "IG\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\u003cspan style='color:rgb(151,116,116)'\u003ei\u003c/span\u003e \u003cspan style='color:rgb(206,89,89)'\u003ereally\u003c/span\u003e \u003cspan style='color:rgb(99,185,99)'\u003edid\u003c/span\u003e \u003cspan style='color:rgb(593,-105,-105)'\u003enot\u003c/span\u003e \u003cspan style='color:rgb(72,240,72)'\u003elike\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003ethis\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e.\u003c/span\u003e \u003cspan style='color:rgb(104,175,104)'\u003eit\u003c/span\u003e \u003cspan style='color:rgb(99,185,99)'\u003ejust\u003c/span\u003e \u003cspan style='color:rgb(90,204,90)'\u003eseemed\u003c/span\u003e \u003cspan style='color:rgb(202,91,91)'\u003eextremely\u003c/span\u003e \u003cspan style='color:rgb(65,254,65)'\u003ewatery\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e.\u003c/span\u003e \u003cspan style='color:rgb(169,107,107)'\u003ei\u003c/span\u003e \u003cspan style='color:rgb(23,337,23)'\u003edont\u003c/span\u003e \u003cspan style='color:rgb(100,184,100)'\u003e'\u003c/span\u003e \u003cspan style='color:rgb(133,125,125)'\u003ethink\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ethis\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ehad\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003eany\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ecarbonation\u003c/span\u003e \u003cspan style='color:rgb(289,47,47)'\u003ewhatsoever\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e.\u003c/span\u003e \u003cspan style='color:rgb(217,83,83)'\u003emaybe\u003c/span\u003e \u003cspan style='color:rgb(91,202,91)'\u003eit\u003c/span\u003e \u003cspan style='color:rgb(179,102,102)'\u003ewas\u003c/span\u003e \u003cspan style='color:rgb(23,338,23)'\u003eflat\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e,\u003c/span\u003e \u003cspan style='color:rgb(276,54,54)'\u003ewho\u003c/span\u003e \u003cspan style='color:rgb(204,90,90)'\u003eknows\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e?\u003c/span\u003e \u003cspan style='color:rgb(115,153,115)'\u003ebut\u003c/span\u003e \u003cspan style='color:rgb(143,120,120)'\u003eeven\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003eif\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ei\u003c/span\u003e \u003cspan style='color:rgb(86,212,86)'\u003egot\u003c/span\u003e \u003cspan style='color:rgb(128,128,128)'\u003ea\u003c/span\u003e \u003cspan style='color:rgb(76,232,76)'\u003ebad\u003c/span\u003e \u003cspan style='color:rgb(79,226,79)'\u003ebrew\u003c/span\u003e \u003cspan style='color:rgb(120,143,120)'\u003ei\u003c/span\u003e \u003cspan style='color:rgb(127,130,127)'\u003edo\u003c/span\u003e \u003cspan style='color:rgb(190,97,97)'\u003en't\u003c/span\u003e \u003cspan style='color:rgb(209,87,87)'\u003esee\u003c/span\u003e \u003cspan style='color:rgb(115,153,115)'\u003ehow\u003c/span\u003e \u003cspan style='color:rgb(107,169,107)'\u003ethis\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ewould\u003c/span\u003e \u003cspan style='color:rgb(136,124,124)'\u003epossibly\u003c/span\u003e \u003cspan style='color:rgb(120,143,120)'\u003ebe\u003c/span\u003e \u003cspan style='color:rgb(205,89,89)'\u003esomething\u003c/span\u003e \u003cspan style='color:rgb(124,136,124)'\u003ei\u003c/span\u003e \u003cspan style='color:rgb(108,168,108)'\u003e'd\u003c/span\u003e \u003cspan style='color:rgb(380,2,2)'\u003eget\u003c/span\u003e \u003cspan style='color:rgb(109,165,109)'\u003etime\u003c/span\u003e \u003cspan style='color:rgb(101,181,101)'\u003eand\u003c/span\u003e \u003cspan style='color:rgb(116,151,116)'\u003etime\u003c/span\u003e \u003cspan style='color:rgb(108,167,108)'\u003eagain\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e.\u003c/span\u003e \u003cspan style='color:rgb(161,111,111)'\u003ei\u003c/span\u003e \u003cspan style='color:rgb(132,126,126)'\u003ecould\u003c/span\u003e \u003cspan style='color:rgb(72,239,72)'\u003etaste\u003c/span\u003e \u003cspan style='color:rgb(51,282,51)'\u003ethe\u003c/span\u003e \u003cspan style='color:rgb(-2,387,-2)'\u003ehops\u003c/span\u003e \u003cspan style='color:rgb(241,71,71)'\u003etowards\u003c/span\u003e \u003cspan style='color:rgb(100,183,100)'\u003ethe\u003c/span\u003e \u003cspan style='color:rgb(268,58,58)'\u003emiddle\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e,\u003c/span\u003e \u003cspan style='color:rgb(97,190,97)'\u003ebut\u003c/span\u003e \u003cspan style='color:rgb(153,115,115)'\u003ethe\u003c/span\u003e \u003cspan style='color:rgb(117,149,117)'\u003ebeer\u003c/span\u003e \u003cspan style='color:rgb(82,219,82)'\u003egot\u003c/span\u003e \u003cspan style='color:rgb(90,203,90)'\u003epretty\u003c/span\u003e \u003cspan style='color:rgb(579,-98,-98)'\u003enasty\u003c/span\u003e \u003cspan style='color:rgb(74,236,74)'\u003etowards\u003c/span\u003e \u003cspan style='color:rgb(185,99,99)'\u003ethe\u003c/span\u003e \u003cspan style='color:rgb(7,370,7)'\u003ebottom\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e.\u003c/span\u003e \u003cspan style='color:rgb(129,127,127)'\u003ei\u003c/span\u003e \u003cspan style='color:rgb(97,189,97)'\u003ewould\u003c/span\u003e \u003cspan style='color:rgb(202,91,91)'\u003enever\u003c/span\u003e \u003cspan style='color:rgb(169,107,107)'\u003edrink\u003c/span\u003e \u003cspan style='color:rgb(135,124,124)'\u003ethis\u003c/span\u003e \u003cspan style='color:rgb(102,180,102)'\u003eagain\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e,\u003c/span\u003e \u003cspan style='color:rgb(143,120,120)'\u003eunless\u003c/span\u003e \u003cspan style='color:rgb(110,163,110)'\u003eit\u003c/span\u003e \u003cspan style='color:rgb(119,146,119)'\u003ewas\u003c/span\u003e \u003cspan style='color:rgb(454,-35,-35)'\u003efree\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e.\u003c/span\u003e \u003cspan style='color:rgb(119,145,119)'\u003ei\u003c/span\u003e \u003cspan style='color:rgb(210,87,87)'\u003e'm\u003c/span\u003e \u003cspan style='color:rgb(191,96,96)'\u003ekind\u003c/span\u003e \u003cspan style='color:rgb(103,177,103)'\u003eof\u003c/span\u003e \u003cspan style='color:rgb(212,86,86)'\u003eupset\u003c/span\u003e \u003cspan style='color:rgb(129,127,127)'\u003ei\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ebought\u003c/span\u003e \u003cspan style='color:rgb(127,128,128)'\u003ethis\u003c/span\u003e \u003cspan style='color:rgb(128,127,128)'\u003e.\u003c/span\u003e"
            ],
            "text/plain": [
              "\u003cIPython.core.display.HTML object\u003e"
            ]
          },
          "metadata": {
            "tags": []
          },
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "def _get_features_and_annotation_from_index(index):\n",
        "  return annotation_x[index], annotations[index]\n",
        "features, annotation = _get_features_and_annotation_from_index(10)\n",
        "\n",
        "smug_saliency, smug_base_saliency, ig_saliency = generate_saliency_maps(\n",
        "  features, run_params)\n",
        "highlight_params = []\n",
        "\n",
        "smug_words, smug_attributes = get_words_with_attributes(\n",
        "    features, smug_saliency, tokenizer)\n",
        "highlight_params.append(('Smug', smug_words, smug_attributes))\n",
        "\n",
        "smug_base_words, smug_base_attributes = get_words_with_attributes(\n",
        "    features, smug_base_saliency, tokenizer)\n",
        "highlight_params.append(\n",
        "  ('No_minimization', smug_base_words, smug_base_attributes))\n",
        "\n",
        "ig_words, ig_attributes = get_words_with_attributes(\n",
        "    features, ig_saliency, tokenizer)\n",
        "highlight_params.append(('IG', ig_words, ig_attributes))\n",
        "\n",
        "for method, words, attributes in highlight_params:\n",
        "  print(method)\n",
        "  highlight_text(\n",
        "      annotation=annotation,\n",
        "      aspect=ASPECT,\n",
        "      words=words,\n",
        "      attributes=attributes,\n",
        "      print_html=False)\n",
        "  print('')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dcQvA-IlFlGR"
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "name": "text_saliency.ipynb",
      "provenance": [
        {
          "file_id": "101BUxrUohvPHzm0ThSCotsTnw5aJSk9q",
          "timestamp": 1616167478688
        }
      ]
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.4"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}
